You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2016/04/26 00:02:49 UTC
incubator-apex-core git commit: APEXCORE-439 Added re-deploying of
dependencies of new operators generated as a result of dynamic
re-partitioning
Repository: incubator-apex-core
Updated Branches:
refs/heads/master 9b6f65d75 -> ddb7471ed
APEXCORE-439 Added re-deploying of dependencies of new operators generated as a result of dynamic re-partitioning
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/ddb7471e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/ddb7471e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/ddb7471e
Branch: refs/heads/master
Commit: ddb7471edd37ef228432c7d80e1e118368e68450
Parents: 9b6f65d
Author: Pramod Immaneni <pr...@datatorrent.com>
Authored: Sat Apr 23 10:52:52 2016 -0700
Committer: Pramod Immaneni <pr...@datatorrent.com>
Committed: Mon Apr 25 09:52:34 2016 -0700
----------------------------------------------------------------------
.../stram/plan/physical/PhysicalPlan.java | 15 +++-
.../stram/plan/physical/PhysicalPlanTest.java | 92 ++++++++++++++++++++
2 files changed, 104 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/ddb7471e/engine/src/main/java/com/datatorrent/stram/plan/physical/PhysicalPlan.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/plan/physical/PhysicalPlan.java b/engine/src/main/java/com/datatorrent/stram/plan/physical/PhysicalPlan.java
index 53e4432..a8b2e23 100644
--- a/engine/src/main/java/com/datatorrent/stram/plan/physical/PhysicalPlan.java
+++ b/engine/src/main/java/com/datatorrent/stram/plan/physical/PhysicalPlan.java
@@ -1122,11 +1122,20 @@ public class PhysicalPlan implements Serializable
Set<PTContainer> releaseContainers = Sets.newHashSet();
assignContainers(newContainers, releaseContainers);
updatePartitionsInfoForPersistOperator(this.dag);
- this.undeployOpers.removeAll(newOpers.keySet());
- //make sure all the new operators are included in deploy operator list
- this.deployOpers.addAll(this.newOpers.keySet());
+
+ // redeploy dependencies of the new operators excluding the new operators themselves
+ Set<PTOperator> ndeps = getDependents(this.newOpers.keySet());
+ this.undeployOpers.addAll(ndeps);
+ this.undeployOpers.removeAll(this.newOpers.keySet());
+
// include downstream dependencies of affected operators into redeploy
Set<PTOperator> deployOperators = this.getDependents(this.deployOpers);
+ // include new operators and their dependencies for deployment
+ deployOperators.addAll(ndeps);
+
+ //make sure all the new operators are included in deploy operator list
+ this.deployOpers.addAll(this.newOpers.keySet());
+
ctx.deploy(releaseContainers, this.undeployOpers, newContainers, deployOperators);
this.newOpers.clear();
this.deployOpers.clear();
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/ddb7471e/engine/src/test/java/com/datatorrent/stram/plan/physical/PhysicalPlanTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/plan/physical/PhysicalPlanTest.java b/engine/src/test/java/com/datatorrent/stram/plan/physical/PhysicalPlanTest.java
index 6e0f853..f10905a 100644
--- a/engine/src/test/java/com/datatorrent/stram/plan/physical/PhysicalPlanTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/plan/physical/PhysicalPlanTest.java
@@ -1514,6 +1514,98 @@ public class PhysicalPlanTest
}
+ /**
+ * Test covering scenario when only new partitions are added during dynamic partitioning and there
+ * are no changes to existing partitions and partition mapping
+ */
+ @Test
+ public void testAugmentedDynamicPartitioning()
+ {
+ LogicalPlan dag = new LogicalPlan();
+
+ TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
+ dag.setAttribute(o1, OperatorContext.PARTITIONER, new TestAugmentingPartitioner<TestGeneratorInputOperator>(3));
+ dag.setAttribute(o1, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener)new PartitioningTest.PartitionLoadWatch()));
+ OperatorMeta o1Meta = dag.getMeta(o1);
+
+ GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
+ OperatorMeta o2Meta = dag.getMeta(o2);
+
+ dag.addStream("o1.outport1", o1.outport, o2.inport1);
+
+ int maxContainers = 10;
+ dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
+
+ TestPlanContext ctx = new TestPlanContext();
+ dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
+
+ PhysicalPlan plan = new PhysicalPlan(dag, ctx);
+ Assert.assertEquals("number of containers", 5, plan.getContainers().size());
+
+ List<PTOperator> o1ops = plan.getOperators(o1Meta);
+ Assert.assertEquals("number of o1 operators", 3, o1ops.size());
+
+ List<PTOperator> o2ops = plan.getOperators(o2Meta);
+ Assert.assertEquals("number of o2 operators", 1, o2ops.size());
+
+ List<PTOperator> uops = plan.getMergeOperators(o1Meta);
+
+ Set<PTOperator> expUndeploy = Sets.newLinkedHashSet();
+ expUndeploy.addAll(plan.getOperators(o2Meta));
+ expUndeploy.addAll(uops);
+
+ for (int i = 0; i < 2; ++i) {
+ PartitioningTest.PartitionLoadWatch.put(o1ops.get(i), 1);
+ plan.onStatusUpdate(o1ops.get(i));
+ }
+
+ ctx.backupRequests = 0;
+ ctx.events.remove(0).run();
+
+ Assert.assertEquals("number of containers", 7, plan.getContainers().size());
+
+ Assert.assertEquals("undeployed opertors", expUndeploy, ctx.undeploy);
+ }
+
+ private class TestAugmentingPartitioner<T> implements Partitioner<T>
+ {
+
+ int initalPartitionCount = 1;
+
+ private TestAugmentingPartitioner(int initialPartitionCount)
+ {
+ this.initalPartitionCount = initialPartitionCount;
+ }
+
+ @Override
+ public Collection<Partition<T>> definePartitions(Collection<Partition<T>> partitions, PartitioningContext context)
+ {
+ Collection<Partition<T>> newPartitions = Lists.newArrayList(partitions);
+ int numTotal = partitions.size();
+ Partition<T> first = partitions.iterator().next();
+ if (first.getStats() == null) {
+ // Initial partition
+ numTotal = initalPartitionCount;
+ } else {
+ for (Partition<T> p : partitions) {
+ // Assumption load is non-negative
+ numTotal += p.getLoad();
+ }
+ }
+ T paritionable = first.getPartitionedInstance();
+ for (int i = partitions.size(); i < numTotal; ++i) {
+ newPartitions.add(new DefaultPartition<T>(paritionable));
+ }
+ return newPartitions;
+ }
+
+ @Override
+ public void partitioned(Map<Integer, Partition<T>> partitions)
+ {
+
+ }
+ }
+
@Test
public void testCascadingUnifier()
{