You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/05/24 21:59:13 UTC
falcon git commit: FALCON-1978 Fix flaky unite test
MetadataMappingServicesTests
Repository: falcon
Updated Branches:
refs/heads/master e7ab83618 -> 3fb6e0e75
FALCON-1978 Fix flaky unite test MetadataMappingServicesTests
1. Ordered the tests so that parallel execution will not cause assert failures
2. There was an issue found in EntityGraphRelationshipBuilder that was causing MetadataMappingServicesTests.testOnClusterEntityChange(), but it seems that has been fixed in the trunk.
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Reviewers: "Balu Vellanki <ba...@apache.org>"
Closes #158 from vramachan/FALCON-1978.FixUnitTest
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3fb6e0e7
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3fb6e0e7
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3fb6e0e7
Branch: refs/heads/master
Commit: 3fb6e0e756ca476a0551f232d57c0dca28a024c6
Parents: e7ab836
Author: Venkatesan Ramachandran <vr...@hortonworks.com>
Authored: Tue May 24 14:59:09 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Tue May 24 14:59:09 2016 -0700
----------------------------------------------------------------------
.../metadata/MetadataMappingServiceTest.java | 274 ++++++++++---------
1 file changed, 146 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/3fb6e0e7/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
index 228f522..c0ae5fc 100644
--- a/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
+++ b/common/src/test/java/org/apache/falcon/metadata/MetadataMappingServiceTest.java
@@ -253,7 +253,139 @@ public class MetadataMappingServiceTest {
verifyEntityGraph(RelationshipType.FEED_ENTITY, "Secure");
}
- @Test
+ @Test (dependsOnMethods = "testOnAdd")
+ // @Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
+ public void testOnChange() throws Exception {
+ // shutdown the graph and resurrect for testing
+ service.destroy();
+ service.init();
+
+ long beforeVerticesCount = getVerticesCount(service.getGraph());
+ long beforeEdgesCount = getEdgesCount(service.getGraph());
+
+ // cannot modify cluster, adding a new cluster
+ anotherCluster = addClusterEntity("another-cluster", "east-coast",
+ "classification=another");
+ verifyEntityWasAddedToGraph("another-cluster", RelationshipType.CLUSTER_ENTITY);
+
+ // +3 = cluster, colo, tag, user (but user falcon-user is already added so ignore from count)
+ Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 3);
+ // +3 edges to user, colo and new tag
+ Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 3);
+ }
+
+ @Test(dependsOnMethods = "testOnChange")
+ public void testOnFeedEntityChange() throws Exception {
+ Feed oldFeed = inputFeeds.get(0);
+ Feed newFeed = EntityBuilderTestUtil.buildFeed(oldFeed.getName(), clusterEntity,
+ "classified-as=Secured,source=data-warehouse", "reporting");
+ addStorage(newFeed, Storage.TYPE.FILESYSTEM,
+ "jail://global:00/falcon/impression-feed/20140101");
+
+ long beforeVerticesCount = 0;
+ long beforeEdgesCount = 0;
+
+ try {
+ configStore.initiateUpdate(newFeed);
+
+ beforeVerticesCount = getVerticesCount(service.getGraph());
+ beforeEdgesCount = getEdgesCount(service.getGraph());
+
+ // add cluster
+ org.apache.falcon.entity.v0.feed.Cluster feedCluster =
+ new org.apache.falcon.entity.v0.feed.Cluster();
+ feedCluster.setName(anotherCluster.getName());
+ newFeed.getClusters().getClusters().add(feedCluster);
+
+ configStore.update(EntityType.FEED, newFeed);
+ } finally {
+ configStore.cleanupUpdateInit();
+ }
+
+ verifyUpdatedEdges(newFeed);
+ Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); //+2 = 2 new tags
+ Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2); // +2 = 1 new cluster, 1 new tag
+ }
+
+ @Test(dependsOnMethods = "testOnFeedEntityChange")
+ public void testOnProcessEntityChange() throws Exception {
+ long beforeVerticesCount = getVerticesCount(service.getGraph());
+ long beforeEdgesCount = getEdgesCount(service.getGraph());
+
+ Process oldProcess = processEntity;
+ Process newProcess = EntityBuilderTestUtil.buildProcess(oldProcess.getName(), anotherCluster,
+ null, null);
+ EntityBuilderTestUtil.addProcessWorkflow(newProcess, GENERATE_WORKFLOW_NAME, "2.0.0");
+ EntityBuilderTestUtil.addInput(newProcess, inputFeeds.get(0));
+
+ try {
+ configStore.initiateUpdate(newProcess);
+ configStore.update(EntityType.PROCESS, newProcess);
+ } finally {
+ configStore.cleanupUpdateInit();
+ }
+
+ verifyUpdatedEdges(newProcess);
+ Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 0); // +0, no net new
+ Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount - 6); // -6 = -2 outputs, -1 tag,
+ // -1 cluster, -2 pipelines
+ }
+
+ @Test(dependsOnMethods = "testOnProcessEntityChange")
+ public void testAreSame() throws Exception {
+
+ Inputs inputs1 = new Inputs();
+ Inputs inputs2 = new Inputs();
+ Outputs outputs1 = new Outputs();
+ Outputs outputs2 = new Outputs();
+ // return true when both are null
+ Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
+ Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
+
+ Input i1 = new Input();
+ i1.setName("input1");
+ Input i2 = new Input();
+ i2.setName("input2");
+ Output o1 = new Output();
+ o1.setName("output1");
+ Output o2 = new Output();
+ o2.setName("output2");
+
+ inputs1.getInputs().add(i1);
+ Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
+ outputs1.getOutputs().add(o1);
+ Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
+
+ inputs2.getInputs().add(i1);
+ Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
+ outputs2.getOutputs().add(o1);
+ Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
+ }
+
+ @Test(dependsOnMethods = "testAreSame")
+ public void testOnClusterEntityChange() throws Exception {
+ long beforeVerticesCount = getVerticesCount(service.getGraph());
+ long beforeEdgesCount = getEdgesCount(service.getGraph());
+
+ Cluster oldCluster = clusterEntity;
+ Cluster newCluster = EntityBuilderTestUtil.buildCluster(oldCluster.getName(),
+ "clusterUpdateColo", oldCluster.getTags() + ",clusterUpdateTagKey=clusterUpdateTagVal");
+
+ try {
+ configStore.initiateUpdate(newCluster);
+ configStore.update(EntityType.CLUSTER, newCluster);
+ } finally {
+ configStore.cleanupUpdateInit();
+ }
+
+ Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); // +1 new tag +1 new colo
+ Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1); // +1 new tag edge
+ Vertex newClusterVertex = getEntityVertex(newCluster.getName(), RelationshipType.CLUSTER_ENTITY);
+ verifyVertexForEdge(newClusterVertex, Direction.OUT, RelationshipLabel.CLUSTER_COLO.getName(),
+ "clusterUpdateColo", RelationshipType.COLO.getName());
+ }
+
+ @Test (dependsOnMethods = "testOnClusterEntityChange")
public void testMapLineage() throws Exception {
setup();
@@ -275,7 +407,7 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 40);
}
- @Test
+ @Test (dependsOnMethods = "testMapLineage")
public void testLineageForNoDateInFeedPath() throws Exception {
setupForNoDateInFeedPath();
@@ -304,7 +436,7 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 34);
}
- @Test
+ @Test (dependsOnMethods = "testLineageForNoDateInFeedPath")
public void testLineageForReplication() throws Exception {
setupForLineageReplication();
@@ -344,7 +476,7 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1);
}
- @Test
+ @Test (dependsOnMethods = "testLineageForReplication")
public void testLineageForReplicationForNonGeneratedInstances() throws Exception {
cleanUp();
service.init();
@@ -374,7 +506,7 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 6);
}
- @Test
+ @Test (dependsOnMethods = "testLineageForReplicationForNonGeneratedInstances")
public void testLineageForRetention() throws Exception {
setupForLineageEviction();
// Get the before vertices and edges
@@ -409,7 +541,7 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2);
}
- @Test
+ @Test (dependsOnMethods = "testLineageForRetention")
public void testLineageForRetentionWithNoFeedsEvicted() throws Exception {
cleanUp();
service.init();
@@ -430,59 +562,20 @@ public class MetadataMappingServiceTest {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount);
}
- @Test (dependsOnMethods = "testOnAdd")
- public void testOnChange() throws Exception {
- // shutdown the graph and resurrect for testing
- service.destroy();
- service.init();
- long beforeVerticesCount = getVerticesCount(service.getGraph());
- long beforeEdgesCount = getEdgesCount(service.getGraph());
- // cannot modify cluster, adding a new cluster
- anotherCluster = addClusterEntity("another-cluster", "east-coast",
- "classification=another");
- verifyEntityWasAddedToGraph("another-cluster", RelationshipType.CLUSTER_ENTITY);
+ private String printProps(Vertex ignored) {
- Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 3); // +3 = cluster, colo, tag
- // +3 edges to user, colo and new tag
- Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 3);
+ StringBuilder sb = new StringBuilder();
+ for(String p : ignored.getPropertyKeys()) {
+ sb.append(p).append("->").append(ignored.getProperty(p) + ";");
+ }
+ return sb.toString();
}
- @Test(dependsOnMethods = "testOnChange")
- public void testOnFeedEntityChange() throws Exception {
- Feed oldFeed = inputFeeds.get(0);
- Feed newFeed = EntityBuilderTestUtil.buildFeed(oldFeed.getName(), clusterEntity,
- "classified-as=Secured,source=data-warehouse", "reporting");
- addStorage(newFeed, Storage.TYPE.FILESYSTEM,
- "jail://global:00/falcon/impression-feed/20140101");
-
- long beforeVerticesCount = 0;
- long beforeEdgesCount = 0;
-
- try {
- configStore.initiateUpdate(newFeed);
-
- beforeVerticesCount = getVerticesCount(service.getGraph());
- beforeEdgesCount = getEdgesCount(service.getGraph());
-
- // add cluster
- org.apache.falcon.entity.v0.feed.Cluster feedCluster =
- new org.apache.falcon.entity.v0.feed.Cluster();
- feedCluster.setName(anotherCluster.getName());
- newFeed.getClusters().getClusters().add(feedCluster);
-
- configStore.update(EntityType.FEED, newFeed);
- } finally {
- configStore.cleanupUpdateInit();
- }
- verifyUpdatedEdges(newFeed);
- Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); //+2 = 2 new tags
- Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2); // +2 = 1 new cluster, 1 new tag
- }
- @Test
+ @Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
public void testLineageForTransactionFailure() throws Exception {
cleanUp();
service.init();
@@ -531,62 +624,8 @@ public class MetadataMappingServiceTest {
"Actual does not contain expected: " + actual);
}
- @Test(dependsOnMethods = "testOnFeedEntityChange")
- public void testOnProcessEntityChange() throws Exception {
- long beforeVerticesCount = getVerticesCount(service.getGraph());
- long beforeEdgesCount = getEdgesCount(service.getGraph());
-
- Process oldProcess = processEntity;
- Process newProcess = EntityBuilderTestUtil.buildProcess(oldProcess.getName(), anotherCluster,
- null, null);
- EntityBuilderTestUtil.addProcessWorkflow(newProcess, GENERATE_WORKFLOW_NAME, "2.0.0");
- EntityBuilderTestUtil.addInput(newProcess, inputFeeds.get(0));
-
- try {
- configStore.initiateUpdate(newProcess);
- configStore.update(EntityType.PROCESS, newProcess);
- } finally {
- configStore.cleanupUpdateInit();
- }
-
- verifyUpdatedEdges(newProcess);
- Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 0); // +0, no net new
- Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount - 6); // -6 = -2 outputs, -1 tag,
- // -1 cluster, -2 pipelines
- }
-
- @Test(dependsOnMethods = "testOnProcessEntityChange")
- public void testAreSame() throws Exception {
-
- Inputs inputs1 = new Inputs();
- Inputs inputs2 = new Inputs();
- Outputs outputs1 = new Outputs();
- Outputs outputs2 = new Outputs();
- // return true when both are null
- Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
- Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
-
- Input i1 = new Input();
- i1.setName("input1");
- Input i2 = new Input();
- i2.setName("input2");
- Output o1 = new Output();
- o1.setName("output1");
- Output o2 = new Output();
- o2.setName("output2");
-
- inputs1.getInputs().add(i1);
- Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
- outputs1.getOutputs().add(o1);
- Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
-
- inputs2.getInputs().add(i1);
- Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
- outputs2.getOutputs().add(o1);
- Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
- }
- @Test
+ @Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
public void testLineageForJobCounter() throws Exception {
setupForJobCounters();
WorkflowExecutionContext context = WorkflowExecutionContext.create(getTestMessageArgs(
@@ -606,28 +645,7 @@ public class MetadataMappingServiceTest {
verifyLineageGraphForJobCounters(context);
}
- @Test(dependsOnMethods = "testOnFeedEntityChange")
- public void testOnClusterEntityChange() throws Exception {
- long beforeVerticesCount = getVerticesCount(service.getGraph());
- long beforeEdgesCount = getEdgesCount(service.getGraph());
-
- Cluster oldCluster = clusterEntity;
- Cluster newCluster = EntityBuilderTestUtil.buildCluster(oldCluster.getName(),
- "clusterUpdateColo", oldCluster.getTags() + ",clusterUpdateTagKey=clusterUpdateTagVal");
-
- try {
- configStore.initiateUpdate(newCluster);
- configStore.update(EntityType.CLUSTER, newCluster);
- } finally {
- configStore.cleanupUpdateInit();
- }
- Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); // +1 new tag +1 new colo
- Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1); // +1 new tag edge
- Vertex newClusterVertex = getEntityVertex(newCluster.getName(), RelationshipType.CLUSTER_ENTITY);
- verifyVertexForEdge(newClusterVertex, Direction.OUT, RelationshipLabel.CLUSTER_COLO.getName(),
- "clusterUpdateColo", RelationshipType.COLO.getName());
- }
private void verifyUpdatedEdges(Process newProcess) {
Vertex processVertex = getEntityVertex(newProcess.getName(), RelationshipType.PROCESS_ENTITY);