You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/03/20 11:06:40 UTC
[01/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Repository: flink
Updated Branches:
refs/heads/master 1b6903216 -> 633b0d6a9
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
new file mode 100644
index 0000000..8720aa7
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichJoinFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+/**
+* Tests that validate optimizer choices when using operators that are requesting certain specific execution
+* strategies.
+*/
+@SuppressWarnings("serial")
+public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
+
+ private static final String JOIN_WITH_INVARIANT_NAME = "Test Join Invariant";
+ private static final String JOIN_WITH_SOLUTION_SET = "Test Join SolutionSet";
+ private static final String NEXT_WORKSET_REDUCER_NAME = "Test Reduce Workset";
+ private static final String SOLUTION_DELTA_MAPPER_NAME = "Test Map Delta";
+
+ @Test
+ public void testJavaApiWithDeferredSoltionSetUpdateWithMapper() {
+ try {
+ Plan plan = getJavaTestPlan(false, true);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+ SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
+
+
+ // verify reducer
+ assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
+
+ // currently, the system may partition before or after the mapper
+ ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
+ ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
+
+ assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
+ (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJavaApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
+ try {
+ Plan plan = getJavaTestPlan(false, false);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
+
+ // verify reducer
+ assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
+
+ // verify solution delta
+ assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJavaApiWithDirectSoltionSetUpdate() {
+ try {
+ Plan plan = getJavaTestPlan(true, false);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
+
+ // verify reducer
+ assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
+
+
+ // verify solution delta
+ assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void testRejectPlanIfSolutionSetKeysAndJoinKeysDontMatch() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> solutionSetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Solution Set");
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> worksetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Workset");
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> invariantInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Invariant Input");
+
+ DeltaIteration<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> iter = solutionSetInput.iterateDelta(worksetInput, 100, 1, 2);
+
+
+ DataSet<Tuple3<Long, Long, Long>> result =
+
+ iter.getWorkset().join(invariantInput)
+ .where(1, 2)
+ .equalTo(1, 2)
+ .with(new JoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
+ public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
+ return first;
+ }
+ });
+
+ try {
+ result.join(iter.getSolutionSet())
+ .where(1, 0)
+ .equalTo(0, 2)
+ .with(new JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
+ public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
+ return second;
+ }
+ });
+ fail("The join should be rejected with key type mismatches.");
+ }
+ catch (InvalidProgramException e) {
+ // expected!
+ }
+
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ private Plan getJavaTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> solutionSetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Solution Set");
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> worksetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Workset");
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> invariantInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Invariant Input");
+
+ DeltaIteration<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> iter = solutionSetInput.iterateDelta(worksetInput, 100, 1, 2);
+
+
+ DataSet<Tuple3<Long, Long, Long>> joinedWithSolutionSet =
+
+ iter.getWorkset().join(invariantInput)
+ .where(1, 2)
+ .equalTo(1, 2)
+ .with(new RichJoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
+ public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
+ return first;
+ }
+ })
+ .name(JOIN_WITH_INVARIANT_NAME)
+
+ .join(iter.getSolutionSet())
+ .where(1, 0)
+ .equalTo(1, 2)
+ .with(new RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
+ public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
+ return second;
+ }
+ })
+ .name(JOIN_WITH_SOLUTION_SET)
+ .withForwardedFieldsSecond(joinPreservesSolutionSet ? new String[] {"0->0", "1->1", "2->2" } : null);
+
+ DataSet<Tuple3<Long, Long, Long>> nextWorkset = joinedWithSolutionSet.groupBy(1, 2)
+ .reduceGroup(new RichGroupReduceFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>() {
+ public void reduce(Iterable<Tuple3<Long, Long, Long>> values, Collector<Tuple3<Long, Long, Long>> out) {}
+ })
+ .name(NEXT_WORKSET_REDUCER_NAME)
+ .withForwardedFields("1->1","2->2","0->0");
+
+
+ DataSet<Tuple3<Long, Long, Long>> nextSolutionSet = mapBeforeSolutionDelta ?
+ joinedWithSolutionSet.map(new RichMapFunction<Tuple3<Long, Long, Long>,Tuple3<Long, Long, Long>>() { public Tuple3<Long, Long, Long> map(Tuple3<Long, Long, Long> value) { return value; } })
+ .name(SOLUTION_DELTA_MAPPER_NAME).withForwardedFields("0->0","1->1","2->2") :
+ joinedWithSolutionSet;
+
+ iter.closeWith(nextSolutionSet, nextWorkset)
+ .print();
+
+ return env.createProgramPlan();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
new file mode 100644
index 0000000..23f8897
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CoGroupGlobalPropertiesCompatibilityTest {
+
+ @Test
+ public void checkCompatiblePartitionings() {
+ try {
+ final FieldList keysLeft = new FieldList(1, 4);
+ final FieldList keysRight = new FieldList(3, 1);
+
+ CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
+
+ // test compatible hash partitioning
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setHashPartitioned(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setHashPartitioned(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setHashPartitioned(keysLeft);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setHashPartitioned(keysRight);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test compatible custom partitioning
+ {
+ Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setCustomPartitioned(keysLeft, part);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setCustomPartitioned(keysRight, part);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test custom partitioning matching any partitioning
+ {
+ Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void checkInompatiblePartitionings() {
+ try {
+ final FieldList keysLeft = new FieldList(1);
+ final FieldList keysRight = new FieldList(3);
+
+ final Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+ final Partitioner<Object> part2 = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
+
+ // test incompatible hash with custom partitioning
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setHashPartitioned(keysLeft);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test incompatible custom partitionings
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part2);
+
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
new file mode 100644
index 0000000..e7807c9
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.configuration.Configuration;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class CoGroupOnConflictingPartitioningsTest extends CompilerTestBase {
+
+ @Test
+ public void testRejectCoGroupOnHashAndRangePartitioning() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+
+ Configuration cfg = new Configuration();
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
+
+ input.coGroup(input).where(0).equalTo(0)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
+ .withParameters(cfg)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ try {
+ compileNoStats(p);
+ fail("This should fail with an exception");
+ }
+ catch (CompilerException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
new file mode 100644
index 0000000..839f0a1
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class JoinGlobalPropertiesCompatibilityTest {
+
+ @Test
+ public void checkCompatiblePartitionings() {
+ try {
+ final FieldList keysLeft = new FieldList(1, 4);
+ final FieldList keysRight = new FieldList(3, 1);
+
+ SortMergeJoinDescriptor descr = new SortMergeJoinDescriptor(keysLeft, keysRight);
+
+ // test compatible hash partitioning
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setHashPartitioned(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setHashPartitioned(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setHashPartitioned(keysLeft);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setHashPartitioned(keysRight);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test compatible custom partitioning
+ {
+ Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setCustomPartitioned(keysLeft, part);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setCustomPartitioned(keysRight, part);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test custom partitioning matching any partitioning
+ {
+ Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void checkInompatiblePartitionings() {
+ try {
+ final FieldList keysLeft = new FieldList(1);
+ final FieldList keysRight = new FieldList(3);
+
+ final Partitioner<Object> part = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+ final Partitioner<Object> part2 = new Partitioner<Object>() {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ SortMergeJoinDescriptor descr = new SortMergeJoinDescriptor(keysLeft, keysRight);
+
+ // test incompatible hash with custom partitioning
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setHashPartitioned(keysLeft);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part);
+
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test incompatible custom partitionings
+ {
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setAnyPartitioning(keysLeft);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setAnyPartitioning(keysRight);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setCustomPartitioned(keysLeft, part);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setCustomPartitioned(keysRight, part2);
+
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
new file mode 100644
index 0000000..9171cc7
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.configuration.Configuration;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class JoinOnConflictingPartitioningsTest extends CompilerTestBase {
+
+ @Test
+ public void testRejectJoinOnHashAndRangePartitioning() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+
+ Configuration cfg = new Configuration();
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
+
+ input.join(input).where(0).equalTo(0)
+ .withParameters(cfg)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ try {
+ compileNoStats(p);
+ fail("This should fail with an exception");
+ }
+ catch (CompilerException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
new file mode 100644
index 0000000..2c1574b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.OperatorInformation;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.core.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.flink.api.java.io.TextInputFormat;
+
+public class ChannelTest {
+
+ @Test
+ public void testGetEstimatesNoReplicationFactor() {
+ final long NUM_RECORD = 1001;
+ final long SIZE = 467131;
+
+ DataSourceNode source = getSourceNode();
+ SourcePlanNode planNode = new SourcePlanNode(source, "test node");
+ Channel channel = new Channel(planNode);
+
+ // no estimates here
+ Assert.assertEquals(-1, channel.getEstimatedOutputSize());
+ Assert.assertEquals(-1, channel.getEstimatedNumRecords());
+
+ // set estimates
+ source.setEstimatedNumRecords(NUM_RECORD);
+ source.setEstimatedOutputSize(SIZE);
+ Assert.assertEquals(SIZE, channel.getEstimatedOutputSize());
+ Assert.assertEquals(NUM_RECORD, channel.getEstimatedNumRecords());
+ }
+
+ @Test
+ public void testGetEstimatesWithReplicationFactor() {
+ final long NUM_RECORD = 1001;
+ final long SIZE = 467131;
+
+ final int REPLICATION = 23;
+
+ DataSourceNode source = getSourceNode();
+ SourcePlanNode planNode = new SourcePlanNode(source, "test node");
+ Channel channel = new Channel(planNode);
+ channel.setReplicationFactor(REPLICATION);
+
+ // no estimates here
+ Assert.assertEquals(-1, channel.getEstimatedOutputSize());
+ Assert.assertEquals(-1, channel.getEstimatedNumRecords());
+
+ // set estimates
+ source.setEstimatedNumRecords(NUM_RECORD);
+ source.setEstimatedOutputSize(SIZE);
+ Assert.assertEquals(SIZE * REPLICATION, channel.getEstimatedOutputSize());
+ Assert.assertEquals(NUM_RECORD * REPLICATION, channel.getEstimatedNumRecords());
+ }
+
+
+// private static final OptimizerNode getSingleInputNode() {
+// return new MapNode(new MapOperatorBase<String, String, GenericMap<String,String>>(
+// new IdentityMapper<String>(),
+// new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+// "map"));
+// }
+
+ private static final DataSourceNode getSourceNode() {
+ return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>(
+ new TextInputFormat(new Path("/ignored")),
+ new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO),
+ "source"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
new file mode 100644
index 0000000..366d10d
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plandump;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class NumberFormattingTest {
+
+ @Test
+ public void testFormatNumberNoDigit() {
+ assertEquals("0.0", PlanJSONDumpGenerator.formatNumber(0));
+ assertEquals("0.00", PlanJSONDumpGenerator.formatNumber(0.0000000001));
+ assertEquals("-1.0", PlanJSONDumpGenerator.formatNumber(-1.0));
+ assertEquals("1.00", PlanJSONDumpGenerator.formatNumber(1));
+ assertEquals("17.00", PlanJSONDumpGenerator.formatNumber(17));
+ assertEquals("17.44", PlanJSONDumpGenerator.formatNumber(17.44));
+ assertEquals("143.00", PlanJSONDumpGenerator.formatNumber(143));
+ assertEquals("143.40", PlanJSONDumpGenerator.formatNumber(143.4));
+ assertEquals("143.50", PlanJSONDumpGenerator.formatNumber(143.5));
+ assertEquals("143.60", PlanJSONDumpGenerator.formatNumber(143.6));
+ assertEquals("143.45", PlanJSONDumpGenerator.formatNumber(143.45));
+ assertEquals("143.55", PlanJSONDumpGenerator.formatNumber(143.55));
+ assertEquals("143.65", PlanJSONDumpGenerator.formatNumber(143.65));
+ assertEquals("143.66", PlanJSONDumpGenerator.formatNumber(143.655));
+
+ assertEquals("1.13 K", PlanJSONDumpGenerator.formatNumber(1126.0));
+ assertEquals("11.13 K", PlanJSONDumpGenerator.formatNumber(11126.0));
+ assertEquals("118.13 K", PlanJSONDumpGenerator.formatNumber(118126.0));
+
+ assertEquals("1.44 M", PlanJSONDumpGenerator.formatNumber(1435126.0));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
new file mode 100644
index 0000000..7fea8a6
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+public class DummyCoGroupFunction<L, R> extends RichCoGroupFunction<L, R, Tuple2<L, R>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coGroup(Iterable<L> first, Iterable<R> second, Collector<Tuple2<L, R>> out) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
new file mode 100644
index 0000000..6be8a24
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichFlatJoinFunction;
+import org.apache.flink.util.Collector;
+
+public class DummyFlatJoinFunction<T> extends RichFlatJoinFunction<T, T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void join(T first, T second, Collector<T> out) {
+ out.collect(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
new file mode 100644
index 0000000..44d3695
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichReduceFunction;
+
+public class DummyReducer<T> extends RichReduceFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public T reduce(T a, T b) {
+ return a;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
new file mode 100644
index 0000000..0316463
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+public class IdentityFlatMapper<T> implements FlatMapFunction<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(T value, Collector<T> out) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
new file mode 100644
index 0000000..11fd044
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable;
+import org.apache.flink.util.Collector;
+
+
+@Combinable
+public class IdentityGroupReducer<T> extends RichGroupReduceFunction<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterable<T> values, Collector<T> out) {
+ for (T next : values) {
+ out.collect(next);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
new file mode 100644
index 0000000..f335846
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.java.functions.KeySelector;
+
+public class IdentityKeyExtractor<T> implements KeySelector<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public T getKey(T value) {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
new file mode 100644
index 0000000..025b4d8
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichMapFunction;
+
+public class IdentityMapper<T> extends RichMapFunction<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public T map(T value) {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
new file mode 100644
index 0000000..6efbef1
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichMapPartitionFunction;
+import org.apache.flink.util.Collector;
+
+public class IdentityPartitionerMapper<T> extends RichMapPartitionFunction<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void mapPartition(Iterable<T> values, Collector<T> out) {
+ for (T in : values) {
+ out.collect(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
new file mode 100644
index 0000000..39c0e1b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichReduceFunction;
+
+public class SelectOneReducer<T> extends RichReduceFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+ return value1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
new file mode 100644
index 0000000..48d13ca
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.testfunctions;
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable;
+import org.apache.flink.util.Collector;
+
+
+@Combinable
+public class Top1GroupReducer<T> extends RichGroupReduceFunction<T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterable<T> values, Collector<T> out) {
+ out.collect(values.iterator().next());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
new file mode 100644
index 0000000..6a84c44
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.api.java.record.functions.CoGroupFunction;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("deprecation")
+public class DummyCoGroupStub extends CoGroupFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) {
+ while (records1.hasNext()) {
+ out.collect(records1.next());
+ }
+
+ while (records2.hasNext()) {
+ out.collect(records2.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
new file mode 100644
index 0000000..8ee2285
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import org.apache.flink.api.java.record.functions.CrossFunction;
+import org.apache.flink.types.Record;
+
+@SuppressWarnings("deprecation")
+public class DummyCrossStub extends CrossFunction {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Record cross(Record first, Record second) throws Exception {
+ return first;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
new file mode 100644
index 0000000..0c816e7
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.java.record.io.DelimitedInputFormat;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.Record;
+
+public final class DummyInputFormat extends DelimitedInputFormat {
+ private static final long serialVersionUID = 1L;
+
+ private final IntValue integer = new IntValue(1);
+
+ @Override
+ public Record readRecord(Record target, byte[] bytes, int offset, int numBytes) {
+ target.setField(0, this.integer);
+ target.setField(1, this.integer);
+ return target;
+ }
+
+ @Override
+ public FileBaseStatistics getStatistics(BaseStatistics cachedStatistics) {
+ return (cachedStatistics instanceof FileBaseStatistics) ? (FileBaseStatistics) cachedStatistics : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
new file mode 100644
index 0000000..d00be6e
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.java.record.functions.JoinFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsFirstExcept;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("deprecation")
+@ConstantFieldsFirstExcept({})
+public class DummyMatchStub extends JoinFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void join(Record value1, Record value2, Collector<Record> out) throws Exception {
+ out.collect(value1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
new file mode 100644
index 0000000..444b48e
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.java.record.functions.JoinFunction;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("deprecation")
+public class DummyNonPreservingMatchStub extends JoinFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void join(Record value1, Record value2, Collector<Record> out) throws Exception {
+ out.collect(value1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
new file mode 100644
index 0000000..1bbe24c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.util;
+
+
+import org.apache.flink.api.java.record.io.DelimitedOutputFormat;
+import org.apache.flink.types.Record;
+
+
+public final class DummyOutputFormat extends DelimitedOutputFormat {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public int serializeRecord(Record rec, byte[] target) throws Exception {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
new file mode 100644
index 0000000..cccc6cb
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.java.record.functions.MapFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsExcept;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("deprecation")
+@ConstantFieldsExcept({})
+public final class IdentityMap extends MapFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void map(Record record, Collector<Record> out) throws Exception {
+ out.collect(record);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
new file mode 100644
index 0000000..f45745d
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsExcept;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+
+@SuppressWarnings("deprecation")
+@ConstantFieldsExcept({})
+public final class IdentityReduce extends ReduceFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
+ while (records.hasNext()) {
+ out.collect(records.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/resources/log4j-test.properties b/flink-optimizer/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..2fb9345
--- /dev/null
+++ b/flink-optimizer/src/test/resources/log4j-test.properties
@@ -0,0 +1,19 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/resources/log4j.properties b/flink-optimizer/src/test/resources/log4j.properties
new file mode 100644
index 0000000..fa3f937
--- /dev/null
+++ b/flink-optimizer/src/test/resources/log4j.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# This file ensures that tests executed from the IDE show log output
+
+log4j.rootLogger=INFO, console
+
+# Log all infos in the given file
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target = System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/resources/logback-test.xml b/flink-optimizer/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..8b3bb27
--- /dev/null
+++ b/flink-optimizer/src/test/resources/logback-test.xml
@@ -0,0 +1,29 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index a59bde4..bf44058 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -109,7 +109,7 @@ under the License.
<exclude>org.apache.flink:flink-java</exclude>
<exclude>org.apache.flink:flink-scala</exclude>
<exclude>org.apache.flink:flink-runtime</exclude>
- <exclude>org.apache.flink:flink-compiler</exclude>
+ <exclude>org.apache.flink:flink-optimizer</exclude>
<exclude>org.apache.flink:flink-spargel</exclude>
<exclude>org.apache.flink:flink-avro</exclude>
<exclude>org.apache.flink:flink-java-examples</exclude>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 43eec6d..1c89170 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -113,7 +113,7 @@ under the License.
<exclude>org.apache.flink:flink-java</exclude>
<exclude>org.apache.flink:flink-scala</exclude>
<exclude>org.apache.flink:flink-runtime</exclude>
- <exclude>org.apache.flink:flink-compiler</exclude>
+ <exclude>org.apache.flink:flink-optimizer</exclude>
<exclude>org.apache.flink:flink-spargel</exclude>
<exclude>org.apache.flink:flink-avro</exclude>
<exclude>org.apache.flink:flink-java-examples</exclude>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 523237e..f350062 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -47,7 +47,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
+ <artifactId>flink-optimizer</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-staging/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/pom.xml b/flink-staging/flink-streaming/pom.xml
index 43b8181..e7c23db 100644
--- a/flink-staging/flink-streaming/pom.xml
+++ b/flink-staging/flink-streaming/pom.xml
@@ -49,12 +49,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index e8da4e9..467fb44 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -42,7 +42,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
+ <artifactId>flink-optimizer</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 7958b14..1670d85 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -46,7 +46,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
+ <artifactId>flink-optimizer</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e771ad4..11cca1b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,7 @@ under the License.
<module>flink-java</module>
<module>flink-scala</module>
<module>flink-runtime</module>
- <module>flink-compiler</module>
+ <module>flink-optimizer</module>
<module>flink-examples</module>
<module>flink-clients</module>
<module>flink-tests</module>
[36/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TempMode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
deleted file mode 100644
index 0d1dfc9..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-/**
- * Enumeration to indicate the mode of temporarily materializing the data that flows across a connection.
- * Introducing such an artificial dam is sometimes necessary to avoid that a certain data flows deadlock
- * themselves, or as a cache to replay an intermediate result.
- */
-public enum TempMode {
-
- NONE(false, false),
- PIPELINE_BREAKER(false, true),
- CACHED(true, false),
- CACHING_PIPELINE_BREAKER(true, true);
-
- // --------------------------------------------------------------------------------------------
-
- private final boolean cached;
-
- private final boolean breaksPipeline;
-
-
- private TempMode(boolean cached, boolean breaksPipeline) {
- this.cached = cached;
- this.breaksPipeline = breaksPipeline;
- }
-
- public boolean isCached() {
- return cached;
- }
-
- public boolean breaksPipeline() {
- return breaksPipeline;
- }
-
- public TempMode makePipelineBreaker() {
- if (this == NONE) {
- return PIPELINE_BREAKER;
- } else if (this == CACHED) {
- return CACHING_PIPELINE_BREAKER;
- } else {
- return this;
- }
- }
-
- public TempMode makeCached() {
- if (this == NONE) {
- return CACHED;
- } else if (this == PIPELINE_BREAKER) {
- return CACHING_PIPELINE_BREAKER;
- } else {
- return this;
- }
- }
-
-
- public TempMode makeNonCached() {
- if (this == CACHED) {
- return NONE;
- } else if (this == CACHING_PIPELINE_BREAKER) {
- return PIPELINE_BREAKER;
- } else {
- return this;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
deleted file mode 100644
index 39da165..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
+++ /dev/null
@@ -1,747 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.DualInputOperator;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual.GlobalPropertiesPair;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual.LocalPropertiesPair;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-
-import com.google.common.collect.Sets;
-
-/**
- * A node in the optimizer plan that represents a PACT with a two different inputs, such as MATCH or CROSS.
- * The two inputs are not substitutable in their sides.
- */
-public abstract class TwoInputNode extends OptimizerNode {
-
- protected final FieldList keys1; // The set of key fields for the first input
-
- protected final FieldList keys2; // The set of key fields for the second input
-
- protected DagConnection input1; // The first input edge
-
- protected DagConnection input2; // The second input edge
-
- private List<OperatorDescriptorDual> cachedDescriptors;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new node with a single input for the optimizer plan.
- *
- * @param pactContract
- * The PACT that the node represents.
- */
- public TwoInputNode(DualInputOperator<?, ?, ?, ?> pactContract) {
- super(pactContract);
-
- int[] k1 = pactContract.getKeyColumns(0);
- int[] k2 = pactContract.getKeyColumns(1);
-
- this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
- this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
-
- if (this.keys1 != null) {
- if (this.keys2 != null) {
- if (this.keys1.size() != this.keys2.size()) {
- throw new CompilerException("Unequal number of key fields on the two inputs.");
- }
- } else {
- throw new CompilerException("Keys are set on first input, but not on second.");
- }
- } else if (this.keys2 != null) {
- throw new CompilerException("Keys are set on second input, but not on first.");
- }
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public DualInputOperator<?, ?, ?, ?> getOperator() {
- return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
- }
-
- /**
- * Gets the <tt>PactConnection</tt> through which this node receives its <i>first</i> input.
- *
- * @return The first input connection.
- */
- public DagConnection getFirstIncomingConnection() {
- return this.input1;
- }
-
- /**
- * Gets the <tt>PactConnection</tt> through which this node receives its <i>second</i> input.
- *
- * @return The second input connection.
- */
- public DagConnection getSecondIncomingConnection() {
- return this.input2;
- }
-
- public OptimizerNode getFirstPredecessorNode() {
- if(this.input1 != null) {
- return this.input1.getSource();
- } else {
- return null;
- }
- }
-
- public OptimizerNode getSecondPredecessorNode() {
- if(this.input2 != null) {
- return this.input2.getSource();
- } else {
- return null;
- }
- }
-
- @Override
- public List<DagConnection> getIncomingConnections() {
- ArrayList<DagConnection> inputs = new ArrayList<DagConnection>(2);
- inputs.add(input1);
- inputs.add(input2);
- return inputs;
- }
-
-
- @Override
- public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
- // see if there is a hint that dictates which shipping strategy to use for BOTH inputs
- final Configuration conf = getOperator().getParameters();
- ShipStrategyType preSet1 = null;
- ShipStrategyType preSet2 = null;
-
- String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
- if (shipStrategy != null) {
- if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
- preSet1 = preSet2 = ShipStrategyType.FORWARD;
- } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
- preSet1 = preSet2 = ShipStrategyType.BROADCAST;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
- preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
- preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
- preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
- } else {
- throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
- }
- }
-
- // see if there is a hint that dictates which shipping strategy to use for the FIRST input
- shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
- if (shipStrategy != null) {
- if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
- preSet1 = ShipStrategyType.FORWARD;
- } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
- preSet1 = ShipStrategyType.BROADCAST;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
- preSet1 = ShipStrategyType.PARTITION_HASH;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
- preSet1 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
- preSet1 = ShipStrategyType.PARTITION_RANDOM;
- } else {
- throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
- }
- }
-
- // see if there is a hint that dictates which shipping strategy to use for the SECOND input
- shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
- if (shipStrategy != null) {
- if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
- preSet2 = ShipStrategyType.FORWARD;
- } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
- preSet2 = ShipStrategyType.BROADCAST;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
- preSet2 = ShipStrategyType.PARTITION_HASH;
- } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
- preSet2 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
- preSet2 = ShipStrategyType.PARTITION_RANDOM;
- } else {
- throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
- }
- }
-
- // get the predecessors
- DualInputOperator<?, ?, ?, ?> contr = getOperator();
-
- Operator<?> leftPred = contr.getFirstInput();
- Operator<?> rightPred = contr.getSecondInput();
-
- OptimizerNode pred1;
- DagConnection conn1;
- if (leftPred == null) {
- throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
- } else {
- pred1 = contractToNode.get(leftPred);
- conn1 = new DagConnection(pred1, this, defaultExecutionMode);
- if (preSet1 != null) {
- conn1.setShipStrategy(preSet1);
- }
- }
-
- // create the connection and add it
- this.input1 = conn1;
- pred1.addOutgoingConnection(conn1);
-
- OptimizerNode pred2;
- DagConnection conn2;
- if (rightPred == null) {
- throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
- } else {
- pred2 = contractToNode.get(rightPred);
- conn2 = new DagConnection(pred2, this, defaultExecutionMode);
- if (preSet2 != null) {
- conn2.setShipStrategy(preSet2);
- }
- }
-
- // create the connection and add it
- this.input2 = conn2;
- pred2.addOutgoingConnection(conn2);
- }
-
- protected abstract List<OperatorDescriptorDual> getPossibleProperties();
-
- private List<OperatorDescriptorDual> getProperties() {
- if (this.cachedDescriptors == null) {
- this.cachedDescriptors = getPossibleProperties();
- }
- return this.cachedDescriptors;
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- // get what we inherit and what is preserved by our user code
- final InterestingProperties props1 = getInterestingProperties().filterByCodeAnnotations(this, 0);
- final InterestingProperties props2 = getInterestingProperties().filterByCodeAnnotations(this, 1);
-
- // add all properties relevant to this node
- for (OperatorDescriptorDual dpd : getProperties()) {
- for (GlobalPropertiesPair gp : dpd.getPossibleGlobalProperties()) {
- // input 1
- props1.addGlobalProperties(gp.getProperties1());
-
- // input 2
- props2.addGlobalProperties(gp.getProperties2());
- }
- for (LocalPropertiesPair lp : dpd.getPossibleLocalProperties()) {
- // input 1
- props1.addLocalProperties(lp.getProperties1());
-
- // input 2
- props2.addLocalProperties(lp.getProperties2());
- }
- }
- this.input1.setInterestingProperties(props1);
- this.input2.setInterestingProperties(props2);
-
- for (DagConnection conn : getBroadcastConnections()) {
- conn.setInterestingProperties(new InterestingProperties());
- }
- }
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- // check if we have a cached version
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- }
-
- boolean childrenSkippedDueToReplicatedInput = false;
-
- // step down to all producer nodes and calculate alternative plans
- final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
- final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
-
- // calculate alternative sub-plans for predecessor
- final Set<RequestedGlobalProperties> intGlobal1 = this.input1.getInterestingProperties().getGlobalProperties();
- final Set<RequestedGlobalProperties> intGlobal2 = this.input2.getInterestingProperties().getGlobalProperties();
-
- // calculate alternative sub-plans for broadcast inputs
- final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
- List<DagConnection> broadcastConnections = getBroadcastConnections();
- List<String> broadcastConnectionNames = getBroadcastConnectionNames();
-
- for (int i = 0; i < broadcastConnections.size(); i++ ) {
- DagConnection broadcastConnection = broadcastConnections.get(i);
- String broadcastConnectionName = broadcastConnectionNames.get(i);
- List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
-
- // wrap the plan candidates in named channels
- HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
- for (PlanNode plan: broadcastPlanCandidates) {
- final NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
- DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(),
- ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
- c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
- broadcastChannels.add(c);
- }
- broadcastPlanChannels.add(broadcastChannels);
- }
-
- final GlobalPropertiesPair[] allGlobalPairs;
- final LocalPropertiesPair[] allLocalPairs;
- {
- Set<GlobalPropertiesPair> pairsGlob = new HashSet<GlobalPropertiesPair>();
- Set<LocalPropertiesPair> pairsLoc = new HashSet<LocalPropertiesPair>();
- for (OperatorDescriptorDual ods : getProperties()) {
- pairsGlob.addAll(ods.getPossibleGlobalProperties());
- pairsLoc.addAll(ods.getPossibleLocalProperties());
- }
- allGlobalPairs = pairsGlob.toArray(new GlobalPropertiesPair[pairsGlob.size()]);
- allLocalPairs = pairsLoc.toArray(new LocalPropertiesPair[pairsLoc.size()]);
- }
-
- final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
-
- final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
- final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
-
- final int dop = getParallelism();
- final int inDop1 = getFirstPredecessorNode().getParallelism();
- final int inDop2 = getSecondPredecessorNode().getParallelism();
-
- final boolean dopChange1 = dop != inDop1;
- final boolean dopChange2 = dop != inDop2;
-
- final boolean input1breaksPipeline = this.input1.isBreakingPipeline();
- final boolean input2breaksPipeline = this.input2.isBreakingPipeline();
-
- // enumerate all pairwise combination of the children's plans together with
- // all possible operator strategy combination
-
- // create all candidates
- for (PlanNode child1 : subPlans1) {
-
- if (child1.getGlobalProperties().isFullyReplicated()) {
- // fully replicated input is always locally forwarded if DOP is not changed
- if (dopChange1) {
- // can not continue with this child
- childrenSkippedDueToReplicatedInput = true;
- continue;
- } else {
- this.input1.setShipStrategy(ShipStrategyType.FORWARD);
- }
- }
-
- for (PlanNode child2 : subPlans2) {
-
- if (child2.getGlobalProperties().isFullyReplicated()) {
- // fully replicated input is always locally forwarded if DOP is not changed
- if (dopChange2) {
- // can not continue with this child
- childrenSkippedDueToReplicatedInput = true;
- continue;
- } else {
- this.input2.setShipStrategy(ShipStrategyType.FORWARD);
- }
- }
-
- // check that the children go together. that is the case if they build upon the same
- // candidate at the joined branch plan.
- if (!areBranchCompatible(child1, child2)) {
- continue;
- }
-
- for (RequestedGlobalProperties igps1: intGlobal1) {
- // create a candidate channel for the first input. mark it cached, if the connection says so
- final Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
- if (this.input1.getShipStrategy() == null) {
- // free to choose the ship strategy
- igps1.parameterizeChannel(c1, dopChange1, input1Mode, input1breaksPipeline);
-
- // if the DOP changed, make sure that we cancel out properties, unless the
- // ship strategy preserves/establishes them even under changing DOPs
- if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
- c1.getGlobalProperties().reset();
- }
- }
- else {
- // ship strategy fixed by compiler hint
- ShipStrategyType shipType = this.input1.getShipStrategy();
- DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipType, input1breaksPipeline);
- if (this.keys1 != null) {
- c1.setShipStrategy(shipType, this.keys1.toFieldList(), exMode);
- }
- else {
- c1.setShipStrategy(shipType, exMode);
- }
-
- if (dopChange1) {
- c1.adjustGlobalPropertiesForFullParallelismChange();
- }
- }
-
- for (RequestedGlobalProperties igps2: intGlobal2) {
- // create a candidate channel for the first input. mark it cached, if the connection says so
- final Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
- if (this.input2.getShipStrategy() == null) {
- // free to choose the ship strategy
- igps2.parameterizeChannel(c2, dopChange2, input2Mode, input2breaksPipeline);
-
- // if the DOP changed, make sure that we cancel out properties, unless the
- // ship strategy preserves/establishes them even under changing DOPs
- if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
- c2.getGlobalProperties().reset();
- }
- } else {
- // ship strategy fixed by compiler hint
- ShipStrategyType shipType = this.input2.getShipStrategy();
- DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipType, input2breaksPipeline);
- if (this.keys2 != null) {
- c2.setShipStrategy(shipType, this.keys2.toFieldList(), exMode);
- } else {
- c2.setShipStrategy(shipType, exMode);
- }
-
- if (dopChange2) {
- c2.adjustGlobalPropertiesForFullParallelismChange();
- }
- }
-
- /* ********************************************************************
- * NOTE: Depending on how we proceed with different partitioning,
- * we might at some point need a compatibility check between
- * the pairs of global properties.
- * *******************************************************************/
-
- outer:
- for (GlobalPropertiesPair gpp : allGlobalPairs) {
- if (gpp.getProperties1().isMetBy(c1.getGlobalProperties()) &&
- gpp.getProperties2().isMetBy(c2.getGlobalProperties()) )
- {
- for (OperatorDescriptorDual desc : getProperties()) {
- if (desc.areCompatible(gpp.getProperties1(), gpp.getProperties2(),
- c1.getGlobalProperties(), c2.getGlobalProperties()))
- {
- Channel c1Clone = c1.clone();
- c1Clone.setRequiredGlobalProps(gpp.getProperties1());
- c2.setRequiredGlobalProps(gpp.getProperties2());
-
- // we form a valid combination, so create the local candidates
- // for this
- addLocalCandidates(c1Clone, c2, broadcastPlanChannels, igps1, igps2,
- outputPlans, allLocalPairs, estimator);
- break outer;
- }
- }
- }
- }
-
- // break the loop over input2's possible global properties, if the property
- // is fixed via a hint. All the properties are overridden by the hint anyways,
- // so we can stop after the first
- if (this.input2.getShipStrategy() != null) {
- break;
- }
- }
-
- // break the loop over input1's possible global properties, if the property
- // is fixed via a hint. All the properties are overridden by the hint anyways,
- // so we can stop after the first
- if (this.input1.getShipStrategy() != null) {
- break;
- }
- }
- }
- }
-
- if(outputPlans.isEmpty()) {
- if(childrenSkippedDueToReplicatedInput) {
- throw new CompilerException("No plan meeting the requirements could be created @ " + this
- + ". Most likely reason: Invalid use of replicated input.");
- } else {
- throw new CompilerException("No plan meeting the requirements could be created @ " + this
- + ". Most likely reason: Too restrictive plan hints.");
- }
- }
-
- // cost and prune the plans
- for (PlanNode node : outputPlans) {
- estimator.costOperator(node);
- }
- prunePlanAlternatives(outputPlans);
- outputPlans.trimToSize();
-
- this.cachedPlans = outputPlans;
- return outputPlans;
- }
-
- protected void addLocalCandidates(Channel template1, Channel template2, List<Set<? extends NamedChannel>> broadcastPlanChannels,
- RequestedGlobalProperties rgps1, RequestedGlobalProperties rgps2,
- List<PlanNode> target, LocalPropertiesPair[] validLocalCombinations, CostEstimator estimator)
- {
- for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
- final Channel in1 = template1.clone();
- ilp1.parameterizeChannel(in1);
-
- for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
- final Channel in2 = template2.clone();
- ilp2.parameterizeChannel(in2);
-
- for (OperatorDescriptorDual dps: getProperties()) {
- for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
- if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) &&
- lpp.getProperties2().isMetBy(in2.getLocalProperties()) )
- {
- // valid combination
- // for non trivial local properties, we need to check that they are co compatible
- // (such as when some sort order is requested, that both are the same sort order
- if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(),
- in1.getLocalProperties(), in2.getLocalProperties()))
- {
- // copy, because setting required properties and instantiation may
- // change the channels and should not affect prior candidates
- Channel in1Copy = in1.clone();
- in1Copy.setRequiredLocalProps(lpp.getProperties1());
-
- Channel in2Copy = in2.clone();
- in2Copy.setRequiredLocalProps(lpp.getProperties2());
-
- // all right, co compatible
- instantiate(dps, in1Copy, in2Copy, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
- break;
- }
- // else cannot use this pair, fall through the loop and try the next one
- }
- }
- }
- }
- }
- }
-
- protected void instantiate(OperatorDescriptorDual operator, Channel in1, Channel in2,
- List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
- RequestedGlobalProperties globPropsReq1, RequestedGlobalProperties globPropsReq2,
- RequestedLocalProperties locPropsReq1, RequestedLocalProperties locPropsReq2)
- {
- final PlanNode inputSource1 = in1.getSource();
- final PlanNode inputSource2 = in2.getSource();
-
- for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
-
- boolean validCombination = true;
-
- // check whether the broadcast inputs use the same plan candidate at the branching point
- for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
- NamedChannel nc = broadcastChannelsCombination.get(i);
- PlanNode bcSource = nc.getSource();
-
- if (!(areBranchCompatible(bcSource, inputSource1) || areBranchCompatible(bcSource, inputSource2))) {
- validCombination = false;
- break;
- }
-
- // check branch compatibility against all other broadcast variables
- for (int k = 0; k < i; k++) {
- PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
-
- if (!areBranchCompatible(bcSource, otherBcSource)) {
- validCombination = false;
- break;
- }
- }
- }
-
- if (!validCombination) {
- continue;
- }
-
- placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
-
- DualInputPlanNode node = operator.instantiate(in1, in2, this);
- node.setBroadcastInputs(broadcastChannelsCombination);
-
- SemanticProperties props = this.getSemanticProperties();
- GlobalProperties gp1 = in1.getGlobalProperties().clone().filterBySemanticProperties(props, 0);
- GlobalProperties gp2 = in2.getGlobalProperties().clone().filterBySemanticProperties(props, 1);
- GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);
-
- LocalProperties lp1 = in1.getLocalProperties().clone().filterBySemanticProperties(props, 0);
- LocalProperties lp2 = in2.getLocalProperties().clone().filterBySemanticProperties(props, 1);
- LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
-
- node.initProperties(combined, locals);
- node.updatePropertiesWithUniqueSets(getUniqueFields());
- target.add(node);
- }
- }
-
- protected void placePipelineBreakersIfNecessary(DriverStrategy strategy, Channel in1, Channel in2) {
- // before we instantiate, check for deadlocks by tracing back to the open branches and checking
- // whether either no input, or all of them have a dam
- if (this.hereJoinedBranches != null && this.hereJoinedBranches.size() > 0) {
- boolean someDamOnLeftPaths = false;
- boolean damOnAllLeftPaths = true;
- boolean someDamOnRightPaths = false;
- boolean damOnAllRightPaths = true;
-
- if (strategy.firstDam() == DamBehavior.FULL_DAM || in1.getLocalStrategy().dams() || in1.getTempMode().breaksPipeline()) {
- someDamOnLeftPaths = true;
- } else {
- for (OptimizerNode brancher : this.hereJoinedBranches) {
- PlanNode candAtBrancher = in1.getSource().getCandidateAtBranchPoint(brancher);
-
- // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
- if (candAtBrancher == null) {
- continue;
- }
-
- SourceAndDamReport res = in1.getSource().hasDamOnPathDownTo(candAtBrancher);
- if (res == NOT_FOUND) {
- throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
- } else if (res == FOUND_SOURCE) {
- damOnAllLeftPaths = false;
- } else if (res == FOUND_SOURCE_AND_DAM) {
- someDamOnLeftPaths = true;
- } else {
- throw new CompilerException();
- }
- }
- }
-
- if (strategy.secondDam() == DamBehavior.FULL_DAM || in2.getLocalStrategy().dams() || in2.getTempMode().breaksPipeline()) {
- someDamOnRightPaths = true;
- } else {
- for (OptimizerNode brancher : this.hereJoinedBranches) {
- PlanNode candAtBrancher = in2.getSource().getCandidateAtBranchPoint(brancher);
-
- // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
- if (candAtBrancher == null) {
- continue;
- }
-
- SourceAndDamReport res = in2.getSource().hasDamOnPathDownTo(candAtBrancher);
- if (res == NOT_FOUND) {
- throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
- } else if (res == FOUND_SOURCE) {
- damOnAllRightPaths = false;
- } else if (res == FOUND_SOURCE_AND_DAM) {
- someDamOnRightPaths = true;
- } else {
- throw new CompilerException();
- }
- }
- }
-
- // okay combinations are both all dam or both no dam
- if ( (damOnAllLeftPaths & damOnAllRightPaths) | (!someDamOnLeftPaths & !someDamOnRightPaths) ) {
- // good, either both materialize already on the way, or both fully pipeline
- } else {
- if (someDamOnLeftPaths & !damOnAllRightPaths) {
- // right needs a pipeline breaker
- in2.setTempMode(in2.getTempMode().makePipelineBreaker());
- }
-
- if (someDamOnRightPaths & !damOnAllLeftPaths) {
- // right needs a pipeline breaker
- in1.setTempMode(in1.getTempMode().makePipelineBreaker());
- }
- }
- }
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- // handle the data flow branching for the regular inputs
- addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
- addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
-
- List<UnclosedBranchDescriptor> result1 = getFirstPredecessorNode().getBranchesForParent(getFirstIncomingConnection());
- List<UnclosedBranchDescriptor> result2 = getSecondPredecessorNode().getBranchesForParent(getSecondIncomingConnection());
-
- ArrayList<UnclosedBranchDescriptor> inputsMerged = new ArrayList<UnclosedBranchDescriptor>();
- mergeLists(result1, result2, inputsMerged, true);
-
- // handle the data flow branching for the broadcast inputs
- List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(inputsMerged);
-
- this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return getOperator().getSemanticProperties();
- }
-
- // --------------------------------------------------------------------------------------------
- // Miscellaneous
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void accept(Visitor<OptimizerNode> visitor) {
- if (visitor.preVisit(this)) {
- if (this.input1 == null || this.input2 == null) {
- throw new CompilerException();
- }
-
- getFirstPredecessorNode().accept(visitor);
- getSecondPredecessorNode().accept(visitor);
-
- for (DagConnection connection : getBroadcastConnections()) {
- connection.getSource().accept(visitor);
- }
-
- visitor.postVisit(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
deleted file mode 100644
index 45ecdac..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-
-public class UnaryOperatorNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> operator;
-
- private final String name;
-
-
-
- public UnaryOperatorNode(String name, FieldSet keys, OperatorDescriptorSingle ... operators) {
- this(name, keys, Arrays.asList(operators));
- }
-
- public UnaryOperatorNode(String name, FieldSet keys, List<OperatorDescriptorSingle> operators) {
- super(keys);
-
- this.operator = operators;
- this.name = name;
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.operator;
- }
-
- @Override
- public String getName() {
- return this.name;
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // we have no estimates by default
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
deleted file mode 100644
index e85f289..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.operators.SolutionSetDeltaOperator;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
-import org.apache.flink.optimizer.util.NoOpBinaryUdfOp;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.types.Nothing;
-import org.apache.flink.util.Visitor;
-
-/**
- * A node in the optimizer's program representation for a workset iteration.
- */
-public class WorksetIterationNode extends TwoInputNode implements IterationNode {
-
- private static final int DEFAULT_COST_WEIGHT = 20;
-
-
- private final FieldList solutionSetKeyFields;
-
- private final GlobalProperties partitionedProperties;
-
- private final List<OperatorDescriptorDual> dataProperties;
-
- private SolutionSetNode solutionSetNode;
-
- private WorksetNode worksetNode;
-
- private OptimizerNode solutionSetDelta;
-
- private OptimizerNode nextWorkset;
-
- private DagConnection solutionSetDeltaRootConnection;
-
- private DagConnection nextWorksetRootConnection;
-
- private SingleRootJoiner singleRoot;
-
- private boolean solutionDeltaImmediatelyAfterSolutionJoin;
-
- private final int costWeight;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new node with a single input for the optimizer plan.
- *
- * @param iteration The iteration operator that the node represents.
- */
- public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
- super(iteration);
-
- final int[] ssKeys = iteration.getSolutionSetKeyFields();
- if (ssKeys == null || ssKeys.length == 0) {
- throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
- }
- this.solutionSetKeyFields = new FieldList(ssKeys);
- this.partitionedProperties = new GlobalProperties();
- this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
-
- int weight = iteration.getMaximumNumberOfIterations() > 0 ?
- iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
-
- if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
- weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
- }
- this.costWeight = weight;
-
- this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
- }
-
- // --------------------------------------------------------------------------------------------
-
- public DeltaIterationBase<?, ?> getIterationContract() {
- return (DeltaIterationBase<?, ?>) getOperator();
- }
-
- public SolutionSetNode getSolutionSetNode() {
- return this.solutionSetNode;
- }
-
- public WorksetNode getWorksetNode() {
- return this.worksetNode;
- }
-
- public OptimizerNode getNextWorkset() {
- return this.nextWorkset;
- }
-
- public OptimizerNode getSolutionSetDelta() {
- return this.solutionSetDelta;
- }
-
- public void setPartialSolution(SolutionSetNode solutionSetNode, WorksetNode worksetNode) {
- if (this.solutionSetNode != null || this.worksetNode != null) {
- throw new IllegalStateException("Error: Initializing WorksetIterationNode multiple times.");
- }
- this.solutionSetNode = solutionSetNode;
- this.worksetNode = worksetNode;
- }
-
- public void setNextPartialSolution(OptimizerNode solutionSetDelta, OptimizerNode nextWorkset,
- ExecutionMode executionMode) {
-
- // check whether the next partial solution is itself the join with
- // the partial solution (so we can potentially do direct updates)
- if (solutionSetDelta instanceof TwoInputNode) {
- TwoInputNode solutionDeltaTwoInput = (TwoInputNode) solutionSetDelta;
- if (solutionDeltaTwoInput.getFirstPredecessorNode() == this.solutionSetNode ||
- solutionDeltaTwoInput.getSecondPredecessorNode() == this.solutionSetNode)
- {
- this.solutionDeltaImmediatelyAfterSolutionJoin = true;
- }
- }
-
- // there needs to be at least one node in the workset path, so
- // if the next workset is equal to the workset, we need to inject a no-op node
- if (nextWorkset == worksetNode || nextWorkset instanceof BinaryUnionNode) {
- NoOpNode noop = new NoOpNode();
- noop.setDegreeOfParallelism(getParallelism());
-
- DagConnection noOpConn = new DagConnection(nextWorkset, noop, executionMode);
- noop.setIncomingConnection(noOpConn);
- nextWorkset.addOutgoingConnection(noOpConn);
-
- nextWorkset = noop;
- }
-
- // attach an extra node to the solution set delta for the cases where we need to repartition
- UnaryOperatorNode solutionSetDeltaUpdateAux = new UnaryOperatorNode("Solution-Set Delta", getSolutionSetKeyFields(),
- new SolutionSetDeltaOperator(getSolutionSetKeyFields()));
- solutionSetDeltaUpdateAux.setDegreeOfParallelism(getParallelism());
-
- DagConnection conn = new DagConnection(solutionSetDelta, solutionSetDeltaUpdateAux, executionMode);
- solutionSetDeltaUpdateAux.setIncomingConnection(conn);
- solutionSetDelta.addOutgoingConnection(conn);
-
- this.solutionSetDelta = solutionSetDeltaUpdateAux;
- this.nextWorkset = nextWorkset;
-
- this.singleRoot = new SingleRootJoiner();
- this.solutionSetDeltaRootConnection = new DagConnection(solutionSetDeltaUpdateAux,
- this.singleRoot, executionMode);
-
- this.nextWorksetRootConnection = new DagConnection(nextWorkset, this.singleRoot, executionMode);
- this.singleRoot.setInputs(this.solutionSetDeltaRootConnection, this.nextWorksetRootConnection);
-
- solutionSetDeltaUpdateAux.addOutgoingConnection(this.solutionSetDeltaRootConnection);
- nextWorkset.addOutgoingConnection(this.nextWorksetRootConnection);
- }
-
- public int getCostWeight() {
- return this.costWeight;
- }
-
- public TwoInputNode getSingleRootOfStepFunction() {
- return this.singleRoot;
- }
-
- public FieldList getSolutionSetKeyFields() {
- return this.solutionSetKeyFields;
- }
-
- public OptimizerNode getInitialSolutionSetPredecessorNode() {
- return getFirstPredecessorNode();
- }
-
- public OptimizerNode getInitialWorksetPredecessorNode() {
- return getSecondPredecessorNode();
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public String getName() {
- return "Workset Iteration";
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new EmptySemanticProperties();
- }
-
- protected void readStubAnnotations() {}
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedOutputSize = getFirstPredecessorNode().getEstimatedOutputSize();
- this.estimatedNumRecords = getFirstPredecessorNode().getEstimatedNumRecords();
- }
-
- // --------------------------------------------------------------------------------------------
- // Properties and Optimization
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return this.dataProperties;
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- // our own solution (the solution set) is always partitioned and this cannot be adjusted
- // depending on what the successor to the workset iteration requests. for that reason,
- // we ignore incoming interesting properties.
-
- // in addition, we need to make 2 interesting property passes, because the root of the step function
- // that computes the next workset needs the interesting properties as generated by the
- // workset source of the step function. the second pass concerns only the workset path.
- // as initial interesting properties, we have the trivial ones for the step function,
- // and partitioned on the solution set key for the solution set delta
-
- RequestedGlobalProperties partitionedProperties = new RequestedGlobalProperties();
- partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
- InterestingProperties partitionedIP = new InterestingProperties();
- partitionedIP.addGlobalProperties(partitionedProperties);
- partitionedIP.addLocalProperties(new RequestedLocalProperties());
-
- this.nextWorksetRootConnection.setInterestingProperties(new InterestingProperties());
- this.solutionSetDeltaRootConnection.setInterestingProperties(partitionedIP.clone());
-
- InterestingPropertyVisitor ipv = new InterestingPropertyVisitor(estimator);
- this.nextWorkset.accept(ipv);
- this.solutionSetDelta.accept(ipv);
-
- // take the interesting properties of the partial solution and add them to the root interesting properties
- InterestingProperties worksetIntProps = this.worksetNode.getInterestingProperties();
- InterestingProperties intProps = new InterestingProperties();
- intProps.getGlobalProperties().addAll(worksetIntProps.getGlobalProperties());
- intProps.getLocalProperties().addAll(worksetIntProps.getLocalProperties());
-
- // clear all interesting properties to prepare the second traversal
- this.nextWorksetRootConnection.clearInterestingProperties();
- this.nextWorkset.accept(InterestingPropertiesClearer.INSTANCE);
-
- // 2nd pass
- this.nextWorksetRootConnection.setInterestingProperties(intProps);
- this.nextWorkset.accept(ipv);
-
- // now add the interesting properties of the workset to the workset input
- final InterestingProperties inProps = this.worksetNode.getInterestingProperties().clone();
- inProps.addGlobalProperties(new RequestedGlobalProperties());
- inProps.addLocalProperties(new RequestedLocalProperties());
- this.input2.setInterestingProperties(inProps);
-
- // the partial solution must be hash partitioned, so it has only that as interesting properties
- this.input1.setInterestingProperties(partitionedIP);
- }
-
- @Override
- public void clearInterestingProperties() {
- super.clearInterestingProperties();
-
- this.nextWorksetRootConnection.clearInterestingProperties();
- this.solutionSetDeltaRootConnection.clearInterestingProperties();
-
- this.nextWorkset.accept(InterestingPropertiesClearer.INSTANCE);
- this.solutionSetDelta.accept(InterestingPropertiesClearer.INSTANCE);
- }
-
- @Override
- protected void instantiate(OperatorDescriptorDual operator, Channel solutionSetIn, Channel worksetIn,
- List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
- RequestedGlobalProperties globPropsReqSolutionSet, RequestedGlobalProperties globPropsReqWorkset,
- RequestedLocalProperties locPropsReqSolutionSet, RequestedLocalProperties locPropsReqWorkset)
- {
- // check for pipeline breaking using hash join with build on the solution set side
- placePipelineBreakersIfNecessary(DriverStrategy.HYBRIDHASH_BUILD_FIRST, solutionSetIn, worksetIn);
-
- // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
- // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
- // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
- // we have a fitting candidate for the step function (often, work is pushed out of the step function).
- // Among the candidates of the step function, we keep only those that meet the requested properties of the
- // current candidate initial partial solution. That makes sure these properties exist at the beginning of
- // every iteration.
-
- // 1) Because we enumerate multiple times, we may need to clean the cached plans
- // before starting another enumeration
- this.nextWorkset.accept(PlanCacheCleaner.INSTANCE);
- this.solutionSetDelta.accept(PlanCacheCleaner.INSTANCE);
-
- // 2) Give the partial solution the properties of the current candidate for the initial partial solution
- // This concerns currently only the workset.
- this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn);
- this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn);
-
- final SolutionSetPlanNode sspn = this.solutionSetNode.getCurrentSolutionSetPlanNode();
- final WorksetPlanNode wspn = this.worksetNode.getCurrentWorksetPlanNode();
-
- // 3) Get the alternative plans
- List<PlanNode> solutionSetDeltaCandidates = this.solutionSetDelta.getAlternativePlans(estimator);
- List<PlanNode> worksetCandidates = this.nextWorkset.getAlternativePlans(estimator);
-
- // 4) Throw away all that are not compatible with the properties currently requested to the
- // initial partial solution
-
- // Make sure that the workset candidates fulfill the input requirements
- {
- List<PlanNode> newCandidates = new ArrayList<PlanNode>();
-
- for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
- PlanNode candidate = planDeleter.next();
-
- GlobalProperties atEndGlobal = candidate.getGlobalProperties();
- LocalProperties atEndLocal = candidate.getLocalProperties();
-
- FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn,
- atEndGlobal, atEndLocal);
-
- if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
- ; // depends only through broadcast variable on the workset solution
- }
- else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
- // attach a no-op node through which we create the properties of the original input
- Channel toNoOp = new Channel(candidate);
- globPropsReqWorkset.parameterizeChannel(toNoOp, false,
- nextWorksetRootConnection.getDataExchangeMode(), false);
- locPropsReqWorkset.parameterizeChannel(toNoOp);
-
- UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties",
- FieldList.EMPTY_LIST);
-
- rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
-
- SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(
- rebuildWorksetPropertiesNode, "Rebuild Workset Properties",
- toNoOp, DriverStrategy.UNARY_NO_OP);
- rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(),
- toNoOp.getLocalProperties());
- estimator.costOperator(rebuildWorksetPropertiesPlanNode);
-
- GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
- LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
-
- if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
- FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(
- wspn, atEndGlobalModified, atEndLocalModified);
- if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
- newCandidates.add(rebuildWorksetPropertiesPlanNode);
- }
- }
-
- // remove the original operator and add the modified candidate
- planDeleter.remove();
-
- }
- }
-
- worksetCandidates.addAll(newCandidates);
- }
-
- if (worksetCandidates.isEmpty()) {
- return;
- }
-
- // sanity check the solution set delta
- for (PlanNode solutionSetDeltaCandidate : solutionSetDeltaCandidates) {
- SingleInputPlanNode candidate = (SingleInputPlanNode) solutionSetDeltaCandidate;
- GlobalProperties gp = candidate.getGlobalProperties();
-
- if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null ||
- !gp.getPartitioningFields().equals(this.solutionSetKeyFields)) {
- throw new CompilerException("Bug: The solution set delta is not partitioned.");
- }
- }
-
- // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
-
- final GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(this.solutionSetKeyFields);
- gp.addUniqueFieldCombination(this.solutionSetKeyFields);
-
- LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
-
- // take all combinations of solution set delta and workset plans
- for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
- for (PlanNode worksetCandidate : worksetCandidates) {
- // check whether they have the same operator at their latest branching point
- if (this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
-
- SingleInputPlanNode siSolutionDeltaCandidate = (SingleInputPlanNode) solutionSetCandidate;
- boolean immediateDeltaUpdate;
-
- // check whether we need a dedicated solution set delta operator, or whether we can update on the fly
- if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD &&
- this.solutionDeltaImmediatelyAfterSolutionJoin)
- {
- // we do not need this extra node. we can make the predecessor the delta
- // sanity check the node and connection
- if (siSolutionDeltaCandidate.getDriverStrategy() != DriverStrategy.UNARY_NO_OP ||
- siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE)
- {
- throw new CompilerException("Invalid Solution set delta node.");
- }
-
- solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource();
- immediateDeltaUpdate = true;
- } else {
- // was not partitioned, we need to keep this node.
- // mark that we materialize the input
- siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
- immediateDeltaUpdate = false;
- }
-
- WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this,
- "WorksetIteration ("+this.getOperator().getName()+")", solutionSetIn,
- worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
- wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
- wsNode.initProperties(gp, lp);
- target.add(wsNode);
- }
- }
- }
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- // IMPORTANT: First compute closed branches from the two inputs
- // we need to do this because the runtime iteration head effectively joins
- addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
- addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
-
- List<UnclosedBranchDescriptor> result1 = getFirstPredecessorNode().getBranchesForParent(getFirstIncomingConnection());
- List<UnclosedBranchDescriptor> result2 = getSecondPredecessorNode().getBranchesForParent(getSecondIncomingConnection());
-
- ArrayList<UnclosedBranchDescriptor> inputsMerged1 = new ArrayList<UnclosedBranchDescriptor>();
- mergeLists(result1, result2, inputsMerged1, true); // this method also sets which branches are joined here (in the head)
-
- addClosedBranches(getSingleRootOfStepFunction().closedBranchingNodes);
-
- ArrayList<UnclosedBranchDescriptor> inputsMerged2 = new ArrayList<UnclosedBranchDescriptor>();
- List<UnclosedBranchDescriptor> result3 = getSingleRootOfStepFunction().openBranches;
- mergeLists(inputsMerged1, result3, inputsMerged2, true);
-
- // handle the data flow branching for the broadcast inputs
- List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(inputsMerged2);
-
- this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
- }
-
- // --------------------------------------------------------------------------------------------
- // Iteration Specific Traversals
- // --------------------------------------------------------------------------------------------
-
- public void acceptForStepFunction(Visitor<OptimizerNode> visitor) {
- this.singleRoot.accept(visitor);
- }
-
- // --------------------------------------------------------------------------------------------
- // Utility Classes
- // --------------------------------------------------------------------------------------------
-
- private static final class WorksetOpDescriptor extends OperatorDescriptorDual {
-
- private WorksetOpDescriptor(FieldList solutionSetKeys) {
- super(solutionSetKeys, null);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.NONE;
- }
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
- RequestedGlobalProperties partitionedGp = new RequestedGlobalProperties();
- partitionedGp.setHashPartitioned(this.keys1);
- return Collections.singletonList(new GlobalPropertiesPair(partitionedGp, new RequestedGlobalProperties()));
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- // all properties are possible
- return Collections.singletonList(new LocalPropertiesPair(
- new RequestedLocalProperties(), new RequestedLocalProperties()));
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2) {
- return true;
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2) {
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- throw new UnsupportedOperationException();
- }
- }
-
- public static class SingleRootJoiner extends TwoInputNode {
-
- SingleRootJoiner() {
- super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));
-
- setDegreeOfParallelism(1);
- }
-
- public void setInputs(DagConnection input1, DagConnection input2) {
- this.input1 = input1;
- this.input2 = input2;
- }
-
- @Override
- public String getName() {
- return "Internal Utility Node";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return Collections.emptyList();
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // no estimates are needed here
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
deleted file mode 100644
index 3b05aba..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-
-/**
- * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
- */
-public class WorksetNode extends AbstractPartialSolutionNode {
-
- private final WorksetIterationNode iterationNode;
-
-
- public WorksetNode(WorksetPlaceHolder<?> psph, WorksetIterationNode iterationNode) {
- super(psph);
- this.iterationNode = iterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
- if (this.cachedPlans != null) {
- throw new IllegalStateException();
- } else {
- WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
- this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
- }
- }
-
- public WorksetPlanNode getCurrentWorksetPlanNode() {
- if (this.cachedPlans != null) {
- return (WorksetPlanNode) this.cachedPlans.get(0);
- } else {
- throw new IllegalStateException();
- }
- }
-
- public WorksetIterationNode getIterationNode() {
- return this.iterationNode;
- }
-
- @Override
- public void computeOutputEstimates(DataStatistics statistics) {
- copyEstimates(this.iterationNode.getInitialWorksetPredecessorNode());
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the contract object for this data source node.
- *
- * @return The contract.
- */
- @Override
- public WorksetPlaceHolder<?> getOperator() {
- return (WorksetPlaceHolder<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Workset";
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- DagConnection worksetInput = this.iterationNode.getSecondIncomingConnection();
- OptimizerNode worksetSource = worksetInput.getSource();
-
- addClosedBranches(worksetSource.closedBranchingNodes);
- List<UnclosedBranchDescriptor> fromInput = worksetSource.getBranchesForParent(worksetInput);
- this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
deleted file mode 100644
index 57ba29d..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
+++ /dev/null
@@ -1,500 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class represents global properties of the data at a certain point in the plan.
- * Global properties are properties that describe data across different partitions, such as
- * whether the data is hash partitioned, range partitioned, replicated, etc.
- */
-public class GlobalProperties implements Cloneable {
-
- public static final Logger LOG = LoggerFactory.getLogger(GlobalProperties.class);
-
- private PartitioningProperty partitioning; // the type partitioning
-
- private FieldList partitioningFields; // the fields which are partitioned
-
- private Ordering ordering; // order of the partitioned fields, if it is an ordered (range) range partitioning
-
- private Set<FieldSet> uniqueFieldCombinations;
-
- private Partitioner<?> customPartitioner;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Initializes the global properties with no partitioning.
- */
- public GlobalProperties() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Sets this global properties to represent a hash partitioning.
- *
- * @param partitionedFields The key fields on which the data is hash partitioned.
- */
- public void setHashPartitioned(FieldList partitionedFields) {
- if (partitionedFields == null) {
- throw new NullPointerException();
- }
-
- this.partitioning = PartitioningProperty.HASH_PARTITIONED;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- }
-
-
- public void setRangePartitioned(Ordering ordering) {
- if (ordering == null) {
- throw new NullPointerException();
- }
-
- this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
- this.ordering = ordering;
- this.partitioningFields = ordering.getInvolvedIndexes();
- }
-
- public void setAnyPartitioning(FieldList partitionedFields) {
- if (partitionedFields == null) {
- throw new NullPointerException();
- }
-
- this.partitioning = PartitioningProperty.ANY_PARTITIONING;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- }
-
- public void setRandomPartitioned() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setFullyReplicated() {
- this.partitioning = PartitioningProperty.FULL_REPLICATION;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setForcedRebalanced() {
- this.partitioning = PartitioningProperty.FORCED_REBALANCED;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setCustomPartitioned(FieldList partitionedFields, Partitioner<?> partitioner) {
- if (partitionedFields == null || partitioner == null) {
- throw new NullPointerException();
- }
-
- this.partitioning = PartitioningProperty.CUSTOM_PARTITIONING;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- this.customPartitioner = partitioner;
- }
-
- public void addUniqueFieldCombination(FieldSet fields) {
- if (fields == null) {
- return;
- }
- if (this.uniqueFieldCombinations == null) {
- this.uniqueFieldCombinations = new HashSet<FieldSet>();
- }
- this.uniqueFieldCombinations.add(fields);
- }
-
- public void clearUniqueFieldCombinations() {
- if (this.uniqueFieldCombinations != null) {
- this.uniqueFieldCombinations = null;
- }
- }
-
- public Set<FieldSet> getUniqueFieldCombination() {
- return this.uniqueFieldCombinations;
- }
-
- public FieldList getPartitioningFields() {
- return this.partitioningFields;
- }
-
- public Ordering getPartitioningOrdering() {
- return this.ordering;
- }
-
- public PartitioningProperty getPartitioning() {
- return this.partitioning;
- }
-
- public Partitioner<?> getCustomPartitioner() {
- return this.customPartitioner;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public boolean isPartitionedOnFields(FieldSet fields) {
- if (this.partitioning.isPartitionedOnKey() && fields.isValidSubset(this.partitioningFields)) {
- return true;
- } else if (this.uniqueFieldCombinations != null) {
- for (FieldSet set : this.uniqueFieldCombinations) {
- if (fields.isValidSubset(set)) {
- return true;
- }
- }
- return false;
- } else {
- return false;
- }
- }
-
- public boolean isExactlyPartitionedOnFields(FieldList fields) {
- return this.partitioning.isPartitionedOnKey() && fields.isExactMatch(this.partitioningFields);
- }
-
- public boolean matchesOrderedPartitioning(Ordering o) {
- if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
- if (this.ordering.getNumberOfFields() > o.getNumberOfFields()) {
- return false;
- }
-
- for (int i = 0; i < this.ordering.getNumberOfFields(); i++) {
- if (this.ordering.getFieldNumber(i) != o.getFieldNumber(i)) {
- return false;
- }
-
- // if this one request no order, everything is good
- final Order oo = o.getOrder(i);
- final Order to = this.ordering.getOrder(i);
- if (oo != Order.NONE) {
- if (oo == Order.ANY) {
- // if any order is requested, any not NONE order is good
- if (to == Order.NONE) {
- return false;
- }
- } else if (oo != to) {
- // the orders must be equal
- return false;
- }
- }
- }
- return true;
- } else {
- return false;
- }
- }
-
- public boolean isFullyReplicated() {
- return this.partitioning == PartitioningProperty.FULL_REPLICATION;
- }
-
- /**
- * Checks, if the properties in this object are trivial, i.e. only standard values.
- */
- public boolean isTrivial() {
- return partitioning == PartitioningProperty.RANDOM_PARTITIONED;
- }
-
- /**
- * This method resets the properties to a state where no properties are given.
- */
- public void reset() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- this.ordering = null;
- this.partitioningFields = null;
- }
-
- /**
- * Filters these GlobalProperties by the fields that are forwarded to the output
- * as described by the SemanticProperties.
- *
- * @param props The semantic properties holding information about forwarded fields.
- * @param input The index of the input.
- * @return The filtered GlobalProperties
- */
- public GlobalProperties filterBySemanticProperties(SemanticProperties props, int input) {
-
- if (props == null) {
- throw new NullPointerException("SemanticProperties may not be null.");
- }
-
- GlobalProperties gp = new GlobalProperties();
-
- // filter partitioning
- switch(this.partitioning) {
- case RANGE_PARTITIONED:
- // check if ordering is preserved
- Ordering newOrdering = new Ordering();
- for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
- int sourceField = this.ordering.getInvolvedIndexes().get(i);
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
-
- if (targetField == null || targetField.size() == 0) {
- // partitioning is destroyed
- newOrdering = null;
- break;
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
- }
- }
- if(newOrdering != null) {
- gp.partitioning = PartitioningProperty.RANGE_PARTITIONED;
- gp.ordering = newOrdering;
- gp.partitioningFields = newOrdering.getInvolvedIndexes();
- }
- break;
- case HASH_PARTITIONED:
- case ANY_PARTITIONING:
- case CUSTOM_PARTITIONING:
- FieldList newPartitioningFields = new FieldList();
- for (int sourceField : this.partitioningFields) {
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
-
- if (targetField == null || targetField.size() == 0) {
- newPartitioningFields = null;
- break;
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newPartitioningFields = newPartitioningFields.addField(targetField.toArray()[0]);
- }
- }
- if(newPartitioningFields != null) {
- gp.partitioning = this.partitioning;
- gp.partitioningFields = newPartitioningFields;
- gp.customPartitioner = this.customPartitioner;
- }
- break;
- case FORCED_REBALANCED:
- case FULL_REPLICATION:
- case RANDOM_PARTITIONED:
- gp.partitioning = this.partitioning;
- break;
- default:
- throw new RuntimeException("Unknown partitioning type.");
- }
-
- // filter unique field combinations
- if (this.uniqueFieldCombinations != null) {
- Set<FieldSet> newUniqueFieldCombinations = new HashSet<FieldSet>();
- for (FieldSet fieldCombo : this.uniqueFieldCombinations) {
- FieldSet newFieldCombo = new FieldSet();
- for (Integer sourceField : fieldCombo) {
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
-
- if (targetField == null || targetField.size() == 0) {
- newFieldCombo = null;
- break;
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newFieldCombo = newFieldCombo.addField(targetField.toArray()[0]);
- }
- }
- if (newFieldCombo != null) {
- newUniqueFieldCombinations.add(newFieldCombo);
- }
- }
- if(!newUniqueFieldCombinations.isEmpty()) {
- gp.uniqueFieldCombinations = newUniqueFieldCombinations;
- }
- }
-
- return gp;
- }
-
-
- public void parameterizeChannel(Channel channel, boolean globalDopChange,
- ExecutionMode exchangeMode, boolean breakPipeline) {
-
- ShipStrategyType shipType;
- FieldList partitionKeys;
- boolean[] sortDirection;
- Partitioner<?> partitioner;
-
- switch (this.partitioning) {
- case RANDOM_PARTITIONED:
- shipType = globalDopChange ? ShipStrategyType.PARTITION_RANDOM : ShipStrategyType.FORWARD;
- partitionKeys = null;
- sortDirection = null;
- partitioner = null;
- break;
-
- case FULL_REPLICATION:
- shipType = ShipStrategyType.BROADCAST;
- partitionKeys = null;
- sortDirection = null;
- partitioner = null;
- break;
-
- case ANY_PARTITIONING:
- case HASH_PARTITIONED:
- shipType = ShipStrategyType.PARTITION_HASH;
- partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
- sortDirection = null;
- partitioner = null;
- break;
-
- case RANGE_PARTITIONED:
- shipType = ShipStrategyType.PARTITION_RANGE;
- partitionKeys = this.ordering.getInvolvedIndexes();
- sortDirection = this.ordering.getFieldSortDirections();
- partitioner = null;
- break;
-
- case FORCED_REBALANCED:
- shipType = ShipStrategyType.PARTITION_RANDOM;
- partitionKeys = null;
- sortDirection = null;
- partitioner = null;
- break;
-
- case CUSTOM_PARTITIONING:
- shipType = ShipStrategyType.PARTITION_CUSTOM;
- partitionKeys = this.partitioningFields;
- sortDirection = null;
- partitioner = this.customPartitioner;
- break;
-
- default:
- throw new CompilerException("Unsupported partitioning strategy");
- }
-
- DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
- channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((partitioning == null) ? 0 : partitioning.ordinal());
- result = prime * result + ((partitioningFields == null) ? 0 : partitioningFields.hashCode());
- result = prime * result + ((ordering == null) ? 0 : ordering.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj != null && obj instanceof GlobalProperties) {
- final GlobalProperties other = (GlobalProperties) obj;
- return (this.partitioning == other.partitioning)
- && (this.ordering == other.ordering || (this.ordering != null && this.ordering.equals(other.ordering)))
- && (this.partitioningFields == other.partitioningFields ||
- (this.partitioningFields != null && this.partitioningFields.equals(other.partitioningFields)))
- && (this.uniqueFieldCombinations == other.uniqueFieldCombinations ||
- (this.uniqueFieldCombinations != null && this.uniqueFieldCombinations.equals(other.uniqueFieldCombinations)));
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder bld = new StringBuilder(
- "GlobalProperties [partitioning=" + partitioning +
- (this.partitioningFields == null ? "" : ", on fields " + this.partitioningFields) +
- (this.ordering == null ? "" : ", with ordering " + this.ordering));
-
- if (this.uniqueFieldCombinations == null) {
- bld.append(']');
- } else {
- bld.append(" - Unique field groups: ");
- bld.append(this.uniqueFieldCombinations);
- bld.append(']');
- }
- return bld.toString();
- }
-
- @Override
- public GlobalProperties clone() {
- final GlobalProperties newProps = new GlobalProperties();
- newProps.partitioning = this.partitioning;
- newProps.partitioningFields = this.partitioningFields;
- newProps.ordering = this.ordering;
- newProps.customPartitioner = this.customPartitioner;
- newProps.uniqueFieldCombinations = this.uniqueFieldCombinations == null ? null : new HashSet<FieldSet>(this.uniqueFieldCombinations);
- return newProps;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static GlobalProperties combine(GlobalProperties gp1, GlobalProperties gp2) {
- if (gp1.isFullyReplicated()) {
- if (gp2.isFullyReplicated()) {
- return new GlobalProperties();
- } else {
- return gp2;
- }
- } else if (gp2.isFullyReplicated()) {
- return gp1;
- } else if (gp1.ordering != null) {
- return gp1;
- } else if (gp2.ordering != null) {
- return gp2;
- } else if (gp1.partitioningFields != null) {
- return gp1;
- } else if (gp2.partitioningFields != null) {
- return gp2;
- } else if (gp1.uniqueFieldCombinations != null) {
- return gp1;
- } else if (gp2.uniqueFieldCombinations != null) {
- return gp2;
- } else if (gp1.getPartitioning().isPartitioned()) {
- return gp1;
- } else if (gp2.getPartitioning().isPartitioned()) {
- return gp2;
- } else {
- return gp1;
- }
- }
-}
[49/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
new file mode 100644
index 0000000..04bc527
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
@@ -0,0 +1,1578 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plantranslate;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.aggregators.AggregatorRegistry;
+import org.apache.flink.api.common.aggregators.AggregatorWithName;
+import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
+import org.apache.flink.api.common.aggregators.LongSumAggregator;
+import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.iterative.convergence.WorksetEmptyConvergenceCriterion;
+import org.apache.flink.runtime.iterative.task.IterationHeadPactTask;
+import org.apache.flink.runtime.iterative.task.IterationIntermediatePactTask;
+import org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask;
+import org.apache.flink.runtime.iterative.task.IterationTailPactTask;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.InputFormatVertex;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.OutputFormatVertex;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.operators.CoGroupDriver;
+import org.apache.flink.runtime.operators.CoGroupWithSolutionSetFirstDriver;
+import org.apache.flink.runtime.operators.CoGroupWithSolutionSetSecondDriver;
+import org.apache.flink.runtime.operators.DataSinkTask;
+import org.apache.flink.runtime.operators.DataSourceTask;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.JoinWithSolutionSetFirstDriver;
+import org.apache.flink.runtime.operators.JoinWithSolutionSetSecondDriver;
+import org.apache.flink.runtime.operators.MatchDriver;
+import org.apache.flink.runtime.operators.NoOpDriver;
+import org.apache.flink.runtime.operators.RegularPactTask;
+import org.apache.flink.runtime.operators.chaining.ChainedDriver;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Visitor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * This component translates the optimizer's resulting {@link org.apache.flink.optimizer.plan.OptimizedPlan}
+ * to a {@link org.apache.flink.runtime.jobgraph.JobGraph}. The translation is not strictly a one-to-one,
+ * because some nodes from the OptimizedPlan are collapsed into one job vertex.
+ *
+ * This translation does not make any decisions or assumptions. All degrees-of-freedom in the execution
+ * of the job are made by the Optimizer, so that this translation becomes a deterministic mapping.
+ *
+ * The basic method of operation is a top down traversal over the plan graph. On the way down, job vertices
+ * are created for the plan nodes, on the way back up, the nodes connect their predecessors.
+ */
+public class JobGraphGenerator implements Visitor<PlanNode> {
+
+ public static final String MERGE_ITERATION_AUX_TASKS_KEY = "compiler.merge-iteration-aux";
+
+ private static final boolean mergeIterationAuxTasks = GlobalConfiguration.getBoolean(MERGE_ITERATION_AUX_TASKS_KEY, false);
+
+ private static final TaskInChain ALREADY_VISITED_PLACEHOLDER = new TaskInChain(null, null, null);
+
+ // ------------------------------------------------------------------------
+
+ private Map<PlanNode, AbstractJobVertex> vertices; // a map from optimizer nodes to job vertices
+
+ private Map<PlanNode, TaskInChain> chainedTasks; // a map from optimizer nodes to job vertices
+
+ private Map<IterationPlanNode, IterationDescriptor> iterations;
+
+ private List<TaskInChain> chainedTasksInSequence;
+
+ private List<AbstractJobVertex> auxVertices; // auxiliary vertices which are added during job graph generation
+
+ private final int defaultMaxFan;
+
+ private final float defaultSortSpillingThreshold;
+
+ private int iterationIdEnumerator = 1;
+
+ private IterationPlanNode currentIteration; // the current the enclosing iteration
+
+ private List<IterationPlanNode> iterationStack; // stack of enclosing iterations
+
+ private SlotSharingGroup sharingGroup;
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new job graph generator that uses the default values for its resource configuration.
+ */
+ public JobGraphGenerator() {
+ this.defaultMaxFan = ConfigConstants.DEFAULT_SPILLING_MAX_FAN;
+ this.defaultSortSpillingThreshold = ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD;
+ }
+
+ public JobGraphGenerator(Configuration config) {
+ this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY,
+ ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
+ this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
+ ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
+ }
+
+ /**
+ * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a
+ * {@link org.apache.flink.runtime.jobgraph.JobGraph}.
+ *
+ * @param program Optimized plan that is translated into a JobGraph.
+ * @return JobGraph generated frmo the plan.
+ */
+ public JobGraph compileJobGraph(OptimizedPlan program) {
+ this.vertices = new HashMap<PlanNode, AbstractJobVertex>();
+ this.chainedTasks = new HashMap<PlanNode, TaskInChain>();
+ this.chainedTasksInSequence = new ArrayList<TaskInChain>();
+ this.auxVertices = new ArrayList<AbstractJobVertex>();
+ this.iterations = new HashMap<IterationPlanNode, IterationDescriptor>();
+ this.iterationStack = new ArrayList<IterationPlanNode>();
+
+ this.sharingGroup = new SlotSharingGroup();
+
+ // this starts the traversal that generates the job graph
+ program.accept(this);
+
+ // sanity check that we are not somehow in an iteration at the end
+ if (this.currentIteration != null) {
+ throw new CompilerException("The graph translation ended prematurely, leaving an unclosed iteration.");
+ }
+
+ // finalize the iterations
+ for (IterationDescriptor iteration : this.iterations.values()) {
+ if (iteration.getIterationNode() instanceof BulkIterationPlanNode) {
+ finalizeBulkIteration(iteration);
+ } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) {
+ finalizeWorksetIteration(iteration);
+ } else {
+ throw new CompilerException();
+ }
+ }
+
+ // now that the traversal is done, we have the chained tasks write their configs into their
+ // parents' configurations
+ for (TaskInChain tic : this.chainedTasksInSequence) {
+ TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
+ t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
+ }
+
+ // create the job graph object
+ JobGraph graph = new JobGraph(program.getJobName());
+ graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());
+ graph.setAllowQueuedScheduling(false);
+
+ // add vertices to the graph
+ for (AbstractJobVertex vertex : this.vertices.values()) {
+ graph.addVertex(vertex);
+ }
+
+ for (AbstractJobVertex vertex : this.auxVertices) {
+ graph.addVertex(vertex);
+ vertex.setSlotSharingGroup(sharingGroup);
+ }
+
+ // add registered cache file into job configuration
+ for (Entry<String, DistributedCacheEntry> e : program.getOriginalPactPlan().getCachedFiles()) {
+ DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
+ }
+
+ try {
+ InstantiationUtil.writeObjectToConfig(
+ program.getOriginalPactPlan().getExecutionConfig(),
+ graph.getJobConfiguration(),
+ ExecutionConfig.CONFIG_KEY);
+ } catch (IOException e) {
+ throw new RuntimeException("Config object could not be written to Job Configuration: " + e);
+ }
+
+ // release all references again
+ this.vertices = null;
+ this.chainedTasks = null;
+ this.chainedTasksInSequence = null;
+ this.auxVertices = null;
+ this.iterations = null;
+ this.iterationStack = null;
+
+ // return job graph
+ return graph;
+ }
+
+ /**
+ * This methods implements the pre-visiting during a depth-first traversal. It create the job vertex and
+ * sets local strategy.
+ *
+ * @param node
+ * The node that is currently processed.
+ * @return True, if the visitor should descend to the node's children, false if not.
+ * @see org.apache.flink.util.Visitor#preVisit(org.apache.flink.util.Visitable)
+ */
+ @Override
+ public boolean preVisit(PlanNode node) {
+ // check if we have visited this node before. in non-tree graphs, this happens
+ if (this.vertices.containsKey(node) || this.chainedTasks.containsKey(node) || this.iterations.containsKey(node)) {
+ // return false to prevent further descend
+ return false;
+ }
+
+ // the vertex to be created for the current node
+ final AbstractJobVertex vertex;
+ try {
+ if (node instanceof SinkPlanNode) {
+ vertex = createDataSinkVertex((SinkPlanNode) node);
+ }
+ else if (node instanceof SourcePlanNode) {
+ vertex = createDataSourceVertex((SourcePlanNode) node);
+ }
+ else if (node instanceof BulkIterationPlanNode) {
+ BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
+ // for the bulk iteration, we skip creating anything for now. we create the graph
+ // for the step function in the post visit.
+
+ // check that the root of the step function has the same DOP as the iteration.
+ // because the tail must have the same DOP as the head, we can only merge the last
+ // operator with the tail, if they have the same DOP. not merging is currently not
+ // implemented
+ PlanNode root = iterationNode.getRootOfStepFunction();
+ if (root.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("Error: The final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+
+ IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
+ this.iterations.put(iterationNode, descr);
+ vertex = null;
+ }
+ else if (node instanceof WorksetIterationPlanNode) {
+ WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
+
+ // we have the same constraints as for the bulk iteration
+ PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
+ PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
+
+ if (nextWorkSet.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("It is currently not supported that the final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+ if (solutionSetDelta.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("It is currently not supported that the final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+
+ IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
+ this.iterations.put(iterationNode, descr);
+ vertex = null;
+ }
+ else if (node instanceof SingleInputPlanNode) {
+ vertex = createSingleInputVertex((SingleInputPlanNode) node);
+ }
+ else if (node instanceof DualInputPlanNode) {
+ vertex = createDualInputVertex((DualInputPlanNode) node);
+ }
+ else if (node instanceof NAryUnionPlanNode) {
+ // skip the union for now
+ vertex = null;
+ }
+ else if (node instanceof BulkPartialSolutionPlanNode) {
+ // create a head node (or not, if it is merged into its successor)
+ vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
+ }
+ else if (node instanceof SolutionSetPlanNode) {
+ // this represents an access into the solution set index.
+ // we do not create a vertex for the solution set here (we create the head at the workset place holder)
+
+ // we adjust the joins / cogroups that go into the solution set here
+ for (Channel c : node.getOutgoingChannels()) {
+ DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
+ AbstractJobVertex accessingVertex = this.vertices.get(target);
+ TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
+ int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
+
+ // sanity checks
+ if (inputNum == -1) {
+ throw new CompilerException();
+ }
+
+ // adjust the driver
+ if (conf.getDriver().equals(MatchDriver.class)) {
+ conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
+ }
+ else if (conf.getDriver().equals(CoGroupDriver.class)) {
+ conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
+ }
+ else {
+ throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
+ }
+ }
+
+ // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
+ this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
+
+ vertex = null;
+ }
+ else if (node instanceof WorksetPlanNode) {
+ // create the iteration head here
+ vertex = createWorksetIterationHead((WorksetPlanNode) node);
+ }
+ else {
+ throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
+ }
+ }
+ catch (Exception e) {
+ throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
+ }
+
+ // check if a vertex was created, or if it was chained or skipped
+ if (vertex != null) {
+ // set degree of parallelism
+ int pd = node.getParallelism();
+ vertex.setParallelism(pd);
+
+ vertex.setSlotSharingGroup(sharingGroup);
+
+ // check whether this vertex is part of an iteration step function
+ if (this.currentIteration != null) {
+ // check that the task has the same DOP as the iteration as such
+ PlanNode iterationNode = (PlanNode) this.currentIteration;
+ if (iterationNode.getParallelism() < pd) {
+ throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
+ }
+
+ // store the id of the iterations the step functions participate in
+ IterationDescriptor descr = this.iterations.get(this.currentIteration);
+ new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
+ }
+
+ // store in the map
+ this.vertices.put(node, vertex);
+ }
+
+ // returning true causes deeper descend
+ return true;
+ }
+
+ /**
+ * This method implements the post-visit during the depth-first traversal. When the post visit happens,
+ * all of the descendants have been processed, so this method connects all of the current node's
+ * predecessors to the current node.
+ *
+ * @param node
+ * The node currently processed during the post-visit.
+ * @see org.apache.flink.util.Visitor#postVisit(org.apache.flink.util.Visitable) t
+ */
+ @Override
+ public void postVisit(PlanNode node) {
+ try {
+ // --------- check special cases for which we handle post visit differently ----------
+
+ // skip data source node (they have no inputs)
+ // also, do nothing for union nodes, we connect them later when gathering the inputs for a task
+ // solution sets have no input. the initial solution set input is connected when the iteration node is in its postVisit
+ if (node instanceof SourcePlanNode || node instanceof NAryUnionPlanNode || node instanceof SolutionSetPlanNode) {
+ return;
+ }
+
+ // check if we have an iteration. in that case, translate the step function now
+ if (node instanceof IterationPlanNode) {
+ // prevent nested iterations
+ if (node.isOnDynamicPath()) {
+ throw new CompilerException("Nested Iterations are not possible at the moment!");
+ }
+
+ // if we recursively go into an iteration (because the constant path of one iteration contains
+ // another one), we push the current one onto the stack
+ if (this.currentIteration != null) {
+ this.iterationStack.add(this.currentIteration);
+ }
+
+ this.currentIteration = (IterationPlanNode) node;
+ this.currentIteration.acceptForStepFunction(this);
+
+ // pop the current iteration from the stack
+ if (this.iterationStack.isEmpty()) {
+ this.currentIteration = null;
+ } else {
+ this.currentIteration = this.iterationStack.remove(this.iterationStack.size() - 1);
+ }
+
+ // inputs for initial bulk partial solution or initial workset are already connected to the iteration head in the head's post visit.
+ // connect the initial solution set now.
+ if (node instanceof WorksetIterationPlanNode) {
+ // connect the initial solution set
+ WorksetIterationPlanNode wsNode = (WorksetIterationPlanNode) node;
+ AbstractJobVertex headVertex = this.iterations.get(wsNode).getHeadTask();
+ TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ int inputIndex = headConfig.getDriverStrategy().getNumInputs();
+ headConfig.setIterationHeadSolutionSetInputIndex(inputIndex);
+ translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
+ }
+
+ return;
+ }
+
+ final AbstractJobVertex targetVertex = this.vertices.get(node);
+
+
+ // --------- Main Path: Translation of channels ----------
+ //
+ // There are two paths of translation: One for chained tasks (or merged tasks in general),
+ // which do not have their own task vertex. The other for tasks that have their own vertex,
+ // or are the primary task in a vertex (to which the others are chained).
+
+ // check whether this node has its own task, or is merged with another one
+ if (targetVertex == null) {
+ // node's task is merged with another task. it is either chained, of a merged head vertex
+ // from an iteration
+ final TaskInChain chainedTask;
+ if ((chainedTask = this.chainedTasks.get(node)) != null) {
+ // Chained Task. Sanity check first...
+ final Iterator<Channel> inConns = node.getInputs().iterator();
+ if (!inConns.hasNext()) {
+ throw new CompilerException("Bug: Found chained task with no input.");
+ }
+ final Channel inConn = inConns.next();
+
+ if (inConns.hasNext()) {
+ throw new CompilerException("Bug: Found a chained task with more than one input!");
+ }
+ if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
+ throw new CompilerException("Bug: Found a chained task with an input local strategy.");
+ }
+ if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
+ throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
+ }
+
+ AbstractJobVertex container = chainedTask.getContainingVertex();
+
+ if (container == null) {
+ final PlanNode sourceNode = inConn.getSource();
+ container = this.vertices.get(sourceNode);
+ if (container == null) {
+ // predecessor is itself chained
+ container = this.chainedTasks.get(sourceNode).getContainingVertex();
+ if (container == null) {
+ throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
+ }
+ } else {
+ // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
+ new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
+ }
+ chainedTask.setContainingVertex(container);
+ }
+
+ // add info about the input serializer type
+ chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
+
+ // update name of container task
+ String containerTaskName = container.getName();
+ if(containerTaskName.startsWith("CHAIN ")) {
+ container.setName(containerTaskName+" -> "+chainedTask.getTaskName());
+ } else {
+ container.setName("CHAIN "+containerTaskName+" -> "+chainedTask.getTaskName());
+ }
+
+ this.chainedTasksInSequence.add(chainedTask);
+ return;
+ }
+ else if (node instanceof BulkPartialSolutionPlanNode ||
+ node instanceof WorksetPlanNode)
+ {
+ // merged iteration head task. the task that the head is merged with will take care of it
+ return;
+ } else {
+ throw new CompilerException("Bug: Unrecognized merged task vertex.");
+ }
+ }
+
+ // -------- Here, we translate non-chained tasks -------------
+
+
+ if (this.currentIteration != null) {
+ AbstractJobVertex head = this.iterations.get(this.currentIteration).getHeadTask();
+ // the head may still be null if we descend into the static parts first
+ if (head != null) {
+ targetVertex.setStrictlyCoLocatedWith(head);
+ }
+ }
+
+
+ // create the config that will contain all the description of the inputs
+ final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
+
+ // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
+ // enclosing iteration node, because the inputs are the initial inputs to the iteration.
+ final Iterator<Channel> inConns;
+ if (node instanceof BulkPartialSolutionPlanNode) {
+ inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
+ // because the partial solution has its own vertex, is has only one (logical) input.
+ // note this in the task configuration
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
+ } else if (node instanceof WorksetPlanNode) {
+ WorksetPlanNode wspn = (WorksetPlanNode) node;
+ // input that is the initial workset
+ inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
+
+ // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
+ // be input 0 and the solution set will be input 1
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
+ targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
+ } else {
+ inConns = node.getInputs().iterator();
+ }
+ if (!inConns.hasNext()) {
+ throw new CompilerException("Bug: Found a non-source task with no input.");
+ }
+
+ int inputIndex = 0;
+ while (inConns.hasNext()) {
+ Channel input = inConns.next();
+ inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
+ }
+ // broadcast variables
+ int broadcastInputIndex = 0;
+ for (NamedChannel broadcastInput: node.getBroadcastInputs()) {
+ int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true);
+ targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
+ targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex);
+ broadcastInputIndex += broadcastInputIndexDelta;
+ }
+ } catch (Exception e) {
+ throw new CompilerException(
+ "An error occurred while translating the optimized plan to a nephele JobGraph: " + e.getMessage(), e);
+ }
+ }
+
+ private int translateChannel(Channel input, int inputIndex, AbstractJobVertex targetVertex,
+ TaskConfig targetVertexConfig, boolean isBroadcast) throws Exception
+ {
+ final PlanNode inputPlanNode = input.getSource();
+ final Iterator<Channel> allInChannels;
+
+ if (inputPlanNode instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) inputPlanNode).getListOfInputs().iterator();
+ }
+ else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
+ if (this.vertices.get(inputPlanNode) == null) {
+ // merged iteration head
+ final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
+ final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
+
+ // check if the iteration's input is a union
+ if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs().iterator();
+ } else {
+ allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
+ }
+
+ // also, set the index of the gate with the partial solution
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
+ } else {
+ // standalone iteration head
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+ } else if (inputPlanNode instanceof WorksetPlanNode) {
+ if (this.vertices.get(inputPlanNode) == null) {
+ // merged iteration head
+ final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
+ final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
+
+ // check if the iteration's input is a union
+ if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs().iterator();
+ } else {
+ allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
+ }
+
+ // also, set the index of the gate with the partial solution
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
+ } else {
+ // standalone iteration head
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+ } else if (inputPlanNode instanceof SolutionSetPlanNode) {
+ // for now, skip connections with the solution set node, as this is a local index access (later to be parameterized here)
+ // rather than a vertex connection
+ return 0;
+ } else {
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+
+ // check that the type serializer is consistent
+ TypeSerializerFactory<?> typeSerFact = null;
+
+ // accounting for channels on the dynamic path
+ int numChannelsTotal = 0;
+ int numChannelsDynamicPath = 0;
+ int numDynamicSenderTasksTotal = 0;
+
+
+ // expand the channel to all the union channels, in case there is a union operator at its source
+ while (allInChannels.hasNext()) {
+ final Channel inConn = allInChannels.next();
+
+ // sanity check the common serializer
+ if (typeSerFact == null) {
+ typeSerFact = inConn.getSerializer();
+ } else if (!typeSerFact.equals(inConn.getSerializer())) {
+ throw new CompilerException("Conflicting types in union operator.");
+ }
+
+ final PlanNode sourceNode = inConn.getSource();
+ AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
+ TaskConfig sourceVertexConfig;
+
+ if (sourceVertex == null) {
+ // this predecessor is chained to another task or an iteration
+ final TaskInChain chainedTask;
+ final IterationDescriptor iteration;
+ if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
+ // push chained task
+ if (chainedTask.getContainingVertex() == null) {
+ throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
+ }
+ sourceVertex = chainedTask.getContainingVertex();
+ sourceVertexConfig = chainedTask.getTaskConfig();
+ } else if ((iteration = this.iterations.get(sourceNode)) != null) {
+ // predecessor is an iteration
+ sourceVertex = iteration.getHeadTask();
+ sourceVertexConfig = iteration.getHeadFinalResultConfig();
+ } else {
+ throw new CompilerException("Bug: Could not resolve source node for a channel.");
+ }
+ } else {
+ // predecessor is its own vertex
+ sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
+ }
+ DistributionPattern pattern = connectJobVertices(
+ inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
+
+ // accounting on channels and senders
+ numChannelsTotal++;
+ if (inConn.isOnDynamicPath()) {
+ numChannelsDynamicPath++;
+ numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
+ sourceVertex.getParallelism(), targetVertex.getParallelism());
+ }
+ }
+
+ // for the iterations, check that the number of dynamic channels is the same as the number
+ // of channels for this logical input. this condition is violated at the moment, if there
+ // is a union between nodes on the static and nodes on the dynamic path
+ if (numChannelsDynamicPath > 0 && numChannelsTotal != numChannelsDynamicPath) {
+ throw new CompilerException("Error: It is currently not supported to union between dynamic and static path in an iteration.");
+ }
+ if (numDynamicSenderTasksTotal > 0) {
+ if (isBroadcast) {
+ targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
+ } else {
+ targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
+ }
+ }
+
+ // the local strategy is added only once. in non-union case that is the actual edge,
+ // in the union case, it is the edge between union and the target node
+ addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
+ return 1;
+ }
+
+ private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) {
+ if (pattern == DistributionPattern.ALL_TO_ALL) {
+ return numSenders;
+ } else if (pattern == DistributionPattern.POINTWISE) {
+ if (numSenders != numReceivers) {
+ if (numReceivers == 1) {
+ return numSenders;
+ }
+ else if (numSenders == 1) {
+ return 1;
+ }
+ else {
+ throw new CompilerException("Error: A changing degree of parallelism is currently " +
+ "not supported between tasks within an iteration.");
+ }
+ } else {
+ return 1;
+ }
+ } else {
+ throw new CompilerException("Unknown distribution pattern for channels: " + pattern);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Methods for creating individual vertices
+ // ------------------------------------------------------------------------
+
+ private AbstractJobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
+ final String taskName = node.getNodeName();
+ final DriverStrategy ds = node.getDriverStrategy();
+
+ // check, whether chaining is possible
+ boolean chaining = false;
+ {
+ Channel inConn = node.getInput();
+ PlanNode pred = inConn.getSource();
+ chaining = ds.getPushChainDriverClass() != null &&
+ !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged
+ !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways
+ !(pred instanceof WorksetPlanNode) && // workset merges anyways
+ !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
+ inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
+ inConn.getLocalStrategy() == LocalStrategy.NONE &&
+ pred.getOutgoingChannels().size() == 1 &&
+ node.getParallelism() == pred.getParallelism() &&
+ node.getBroadcastInputs().isEmpty();
+
+ // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
+ // in a tail
+ if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
+ node.getOutgoingChannels().size() > 0)
+ {
+ WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
+ if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
+ chaining = false;
+ }
+ }
+ // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
+ if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
+ {
+ BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
+ if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
+ chaining = false;
+ }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
+ wspn.getRootOfTerminationCriterion() == pred)) {
+ chaining = false;
+ }
+ }
+ }
+
+ final AbstractJobVertex vertex;
+ final TaskConfig config;
+
+ if (chaining) {
+ vertex = null;
+ config = new TaskConfig(new Configuration());
+ this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
+ } else {
+ // create task vertex
+ vertex = new AbstractJobVertex(taskName);
+ vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
+
+ config = new TaskConfig(vertex.getConfiguration());
+ config.setDriver(ds.getDriverClass());
+ }
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ // set the driver strategy
+ config.setDriverStrategy(ds);
+ for(int i=0;i<ds.getNumRequiredComparators();i++) {
+ config.setDriverComparator(node.getComparator(i), i);
+ }
+ // assign memory, file-handles, etc.
+ assignDriverResources(node, config);
+ return vertex;
+ }
+
+ private AbstractJobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
+ final String taskName = node.getNodeName();
+ final DriverStrategy ds = node.getDriverStrategy();
+ final AbstractJobVertex vertex = new AbstractJobVertex(taskName);
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+ vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ // set the driver strategy
+ config.setDriver(ds.getDriverClass());
+ config.setDriverStrategy(ds);
+ if (node.getComparator1() != null) {
+ config.setDriverComparator(node.getComparator1(), 0);
+ }
+ if (node.getComparator2() != null) {
+ config.setDriverComparator(node.getComparator2(), 1);
+ }
+ if (node.getPairComparator() != null) {
+ config.setDriverPairComparator(node.getPairComparator());
+ }
+
+ // assign memory, file-handles, etc.
+ assignDriverResources(node, config);
+ return vertex;
+ }
+
+ private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
+ final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+
+ vertex.setInvokableClass(DataSourceTask.class);
+ vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ config.setOutputSerializer(node.getSerializer());
+ return vertex;
+ }
+
+ private AbstractJobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
+ final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName());
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+
+ vertex.setInvokableClass(DataSinkTask.class);
+ vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ return vertex;
+ }
+
+ private AbstractJobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
+ // get the bulk iteration that corresponds to this partial solution node
+ final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
+
+ // check whether we need an individual vertex for the partial solution, or whether we
+ // attach ourselves to the vertex of the parent node. We can combine the head with a node of
+ // the step function, if
+ // 1) There is one parent that the partial solution connects to via a forward pattern and no
+ // local strategy
+ // 2) DOP and the number of subtasks per instance does not change
+ // 3) That successor is not a union
+ // 4) That successor is not itself the last node of the step function
+ // 5) There is no local strategy on the edge for the initial partial solution, as
+ // this translates to a local strategy that would only be executed in the first iteration
+
+ final boolean merge;
+ if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
+ final Channel c = pspn.getOutgoingChannels().get(0);
+ final PlanNode successor = c.getTarget();
+ merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
+ c.getLocalStrategy() == LocalStrategy.NONE &&
+ c.getTempMode() == TempMode.NONE &&
+ successor.getParallelism() == pspn.getParallelism() &&
+ !(successor instanceof NAryUnionPlanNode) &&
+ successor != iteration.getRootOfStepFunction() &&
+ iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
+ } else {
+ merge = false;
+ }
+
+ // create or adopt the head vertex
+ final AbstractJobVertex toReturn;
+ final AbstractJobVertex headVertex;
+ final TaskConfig headConfig;
+ if (merge) {
+ final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
+ headVertex = (AbstractJobVertex) this.vertices.get(successor);
+
+ if (headVertex == null) {
+ throw new CompilerException(
+ "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
+ }
+
+ // reset the vertex type to iteration head
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ toReturn = null;
+ } else {
+ // instantiate the head vertex and give it a no-op driver as the driver strategy.
+ // everything else happens in the post visit, after the input (the initial partial solution)
+ // is connected.
+ headVertex = new AbstractJobVertex("PartialSolution ("+iteration.getNodeName()+")");
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ headConfig.setDriver(NoOpDriver.class);
+ toReturn = headVertex;
+ }
+
+ // create the iteration descriptor and the iteration to it
+ IterationDescriptor descr = this.iterations.get(iteration);
+ if (descr == null) {
+ throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
+ }
+ descr.setHeadTask(headVertex, headConfig);
+
+ return toReturn;
+ }
+
+ private AbstractJobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
+ // get the bulk iteration that corresponds to this partial solution node
+ final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
+
+ // check whether we need an individual vertex for the partial solution, or whether we
+ // attach ourselves to the vertex of the parent node. We can combine the head with a node of
+ // the step function, if
+ // 1) There is one parent that the partial solution connects to via a forward pattern and no
+ // local strategy
+ // 2) DOP and the number of subtasks per instance does not change
+ // 3) That successor is not a union
+ // 4) That successor is not itself the last node of the step function
+ // 5) There is no local strategy on the edge for the initial workset, as
+ // this translates to a local strategy that would only be executed in the first superstep
+
+ final boolean merge;
+ if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
+ final Channel c = wspn.getOutgoingChannels().get(0);
+ final PlanNode successor = c.getTarget();
+ merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
+ c.getLocalStrategy() == LocalStrategy.NONE &&
+ c.getTempMode() == TempMode.NONE &&
+ successor.getParallelism() == wspn.getParallelism() &&
+ !(successor instanceof NAryUnionPlanNode) &&
+ successor != iteration.getNextWorkSetPlanNode() &&
+ iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
+ } else {
+ merge = false;
+ }
+
+ // create or adopt the head vertex
+ final AbstractJobVertex toReturn;
+ final AbstractJobVertex headVertex;
+ final TaskConfig headConfig;
+ if (merge) {
+ final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
+ headVertex = (AbstractJobVertex) this.vertices.get(successor);
+
+ if (headVertex == null) {
+ throw new CompilerException(
+ "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
+ }
+
+ // reset the vertex type to iteration head
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ toReturn = null;
+ } else {
+ // instantiate the head vertex and give it a no-op driver as the driver strategy.
+ // everything else happens in the post visit, after the input (the initial partial solution)
+ // is connected.
+ headVertex = new AbstractJobVertex("IterationHead("+iteration.getNodeName()+")");
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ headConfig.setDriver(NoOpDriver.class);
+ toReturn = headVertex;
+ }
+
+ headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
+
+ // create the iteration descriptor and the iteration to it
+ IterationDescriptor descr = this.iterations.get(iteration);
+ if (descr == null) {
+ throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
+ }
+ descr.setHeadTask(headVertex, headConfig);
+
+ return toReturn;
+ }
+
+ private void assignDriverResources(PlanNode node, TaskConfig config) {
+ final double relativeMem = node.getRelativeMemoryPerSubTask();
+ if (relativeMem > 0) {
+ config.setRelativeMemoryDriver(relativeMem);
+ config.setFilehandlesDriver(this.defaultMaxFan);
+ config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
+ }
+ }
+
+ private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) {
+ if (c.getRelativeMemoryLocalStrategy() > 0) {
+ config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy());
+ config.setFilehandlesInput(inputNum, this.defaultMaxFan);
+ config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Connecting Vertices
+ // ------------------------------------------------------------------------
+
+ /**
+ * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
+ * channel is then the channel into the union node, the local strategy channel the one from the union to the
+ * actual target operator.
+ *
+ * @param channel
+ * @param inputNumber
+ * @param sourceVertex
+ * @param sourceConfig
+ * @param targetVertex
+ * @param targetConfig
+ * @param isBroadcast
+ * @throws CompilerException
+ */
+ private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
+ final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
+ final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
+ throws CompilerException
+ {
+ // ------------ connect the vertices to the job graph --------------
+ final DistributionPattern distributionPattern;
+
+ switch (channel.getShipStrategy()) {
+ case FORWARD:
+ distributionPattern = DistributionPattern.POINTWISE;
+ break;
+ case PARTITION_RANDOM:
+ case BROADCAST:
+ case PARTITION_HASH:
+ case PARTITION_CUSTOM:
+ case PARTITION_RANGE:
+ case PARTITION_FORCED_REBALANCE:
+ distributionPattern = DistributionPattern.ALL_TO_ALL;
+ break;
+ default:
+ throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
+ }
+
+ final ResultPartitionType resultType;
+
+ switch (channel.getDataExchangeMode()) {
+
+ case PIPELINED:
+ resultType = ResultPartitionType.PIPELINED;
+ break;
+
+ case BATCH:
+ // BLOCKING results are currently not supported in closed loop iterations
+ //
+ // See https://issues.apache.org/jira/browse/FLINK-1713 for details
+ resultType = channel.getSource().isOnDynamicPath()
+ ? ResultPartitionType.PIPELINED
+ : ResultPartitionType.BLOCKING;
+ break;
+
+ case PIPELINE_WITH_BATCH_FALLBACK:
+ throw new UnsupportedOperationException("Data exchange mode " +
+ channel.getDataExchangeMode() + " currently not supported.");
+
+ default:
+ throw new UnsupportedOperationException("Unknown data exchange mode.");
+
+ }
+
+ targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
+
+ // -------------- configure the source task's ship strategy strategies in task config --------------
+ final int outputIndex = sourceConfig.getNumOutputs();
+ sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
+ if (outputIndex == 0) {
+ sourceConfig.setOutputSerializer(channel.getSerializer());
+ }
+ if (channel.getShipStrategyComparator() != null) {
+ sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
+ }
+
+ if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
+
+ final DataDistribution dataDistribution = channel.getDataDistribution();
+ if (dataDistribution != null) {
+ sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
+ } else {
+ throw new RuntimeException("Range partitioning requires data distribution");
+ // TODO: inject code and configuration for automatic histogram generation
+ }
+ }
+
+ if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
+ if (channel.getPartitioner() != null) {
+ sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
+ } else {
+ throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
+ }
+ }
+
+ // ---------------- configure the receiver -------------------
+ if (isBroadcast) {
+ targetConfig.addBroadcastInputToGroup(inputNumber);
+ } else {
+ targetConfig.addInputToGroup(inputNumber);
+ }
+ return distributionPattern;
+ }
+
+ private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
+ // serializer
+ if (isBroadcastChannel) {
+ config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
+
+ if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
+ throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
+ } else {
+ return;
+ }
+ } else {
+ config.setInputSerializer(channel.getSerializer(), inputNum);
+ }
+
+ // local strategy
+ if (channel.getLocalStrategy() != LocalStrategy.NONE) {
+ config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
+ if (channel.getLocalStrategyComparator() != null) {
+ config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
+ }
+ }
+
+ assignLocalStrategyResources(channel, config, inputNum);
+
+ // materialization / caching
+ if (channel.getTempMode() != null) {
+ final TempMode tm = channel.getTempMode();
+
+ boolean needsMemory = false;
+ // Don't add a pipeline breaker if the data exchange is already blocking.
+ if (tm.breaksPipeline() && channel.getDataExchangeMode() != DataExchangeMode.BATCH) {
+ config.setInputAsynchronouslyMaterialized(inputNum, true);
+ needsMemory = true;
+ }
+ if (tm.isCached()) {
+ config.setInputCached(inputNum, true);
+ needsMemory = true;
+ }
+
+ if (needsMemory) {
+ // sanity check
+ if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
+ throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
+ }
+ config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
+ }
+ }
+ }
+
+ private void finalizeBulkIteration(IterationDescriptor descr) {
+
+ final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
+ final AbstractJobVertex headVertex = descr.getHeadTask();
+ final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
+
+ // ------------ finalize the head config with the final outputs and the sync gate ------------
+ final int numStepFunctionOuts = headConfig.getNumOutputs();
+ final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
+
+ if (numStepFunctionOuts == 0) {
+ throw new CompilerException("The iteration has no operation inside the step function.");
+ }
+
+ headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
+ headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
+ final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
+ if (relativeMemForBackChannel <= 0) {
+ throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
+ }
+ headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
+
+ // --------------------------- create the sync task ---------------------------
+ final AbstractJobVertex sync = new AbstractJobVertex("Sync(" + bulkNode.getNodeName() + ")");
+ sync.setInvokableClass(IterationSynchronizationSinkTask.class);
+ sync.setParallelism(1);
+ this.auxVertices.add(sync);
+
+ final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
+ syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
+
+ // set the number of iteration / convergence criterion for the sync
+ final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
+ if (maxNumIterations < 1) {
+ throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
+ }
+ syncConfig.setNumberOfIterations(maxNumIterations);
+
+ // connect the sync task
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+
+ // ----------------------------- create the iteration tail ------------------------------
+
+ final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
+ final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
+ final TaskConfig tailConfig;
+
+ AbstractJobVertex rootOfStepFunctionVertex = (AbstractJobVertex) this.vertices.get(rootOfStepFunction);
+ if (rootOfStepFunctionVertex == null) {
+ // last op is chained
+ final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
+ }
+ rootOfStepFunctionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+
+ // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
+ tailConfig = taskInChain.getTaskConfig();
+ } else {
+ tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
+ }
+
+ tailConfig.setIsWorksetUpdate();
+
+ // No following termination criterion
+ if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
+
+ rootOfStepFunctionVertex.setInvokableClass(IterationTailPactTask.class);
+
+ tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
+ }
+
+
+ // create the fake output task for termination criterion, if needed
+ final TaskConfig tailConfigOfTerminationCriterion;
+ // If we have a termination criterion and it is not an intermediate node
+ if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
+ AbstractJobVertex rootOfTerminationCriterionVertex = (AbstractJobVertex) this.vertices.get(rootOfTerminationCriterion);
+
+
+ if (rootOfTerminationCriterionVertex == null) {
+ // last op is chained
+ final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
+ }
+ rootOfTerminationCriterionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+
+ // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
+ tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
+ } else {
+ tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
+ }
+
+ rootOfTerminationCriterionVertex.setInvokableClass(IterationTailPactTask.class);
+ // Hack
+ tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
+ tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
+
+ // tell the head that it needs to wait for the solution set updates
+ headConfig.setWaitForSolutionSetUpdate();
+ }
+
+ // ------------------- register the aggregators -------------------
+ AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
+ Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
+
+ headConfig.addIterationAggregators(allAggregators);
+ syncConfig.addIterationAggregators(allAggregators);
+
+ String convAggName = aggs.getConvergenceCriterionAggregatorName();
+ ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
+
+ if (convCriterion != null || convAggName != null) {
+ if (convCriterion == null) {
+ throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
+ }
+ if (convAggName == null) {
+ throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
+ }
+
+ syncConfig.setConvergenceCriterion(convAggName, convCriterion);
+ }
+ }
+
+ private void finalizeWorksetIteration(IterationDescriptor descr) {
+ final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
+ final AbstractJobVertex headVertex = descr.getHeadTask();
+ final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
+
+ // ------------ finalize the head config with the final outputs and the sync gate ------------
+ {
+ final int numStepFunctionOuts = headConfig.getNumOutputs();
+ final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
+
+ if (numStepFunctionOuts == 0) {
+ throw new CompilerException("The workset iteration has no operation on the workset inside the step function.");
+ }
+
+ headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
+ headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
+ final double relativeMemory = iterNode.getRelativeMemoryPerSubTask();
+ if (relativeMemory <= 0) {
+ throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
+ }
+
+ headConfig.setIsWorksetIteration();
+ headConfig.setRelativeBackChannelMemory(relativeMemory / 2);
+ headConfig.setRelativeSolutionSetMemory(relativeMemory / 2);
+
+ // set the solution set serializer and comparator
+ headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
+ headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
+ }
+
+ // --------------------------- create the sync task ---------------------------
+ final TaskConfig syncConfig;
+ {
+ final AbstractJobVertex sync = new AbstractJobVertex("Sync (" + iterNode.getNodeName() + ")");
+ sync.setInvokableClass(IterationSynchronizationSinkTask.class);
+ sync.setParallelism(1);
+ this.auxVertices.add(sync);
+
+ syncConfig = new TaskConfig(sync.getConfiguration());
+ syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
+
+ // set the number of iteration / convergence criterion for the sync
+ final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
+ if (maxNumIterations < 1) {
+ throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
+ }
+ syncConfig.setNumberOfIterations(maxNumIterations);
+
+ // connect the sync task
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ }
+
+ // ----------------------------- create the iteration tails -----------------------------
+ // ----------------------- for next workset and solution set delta-----------------------
+
+ {
+ // we have three possible cases:
+ // 1) Two tails, one for workset update, one for solution set update
+ // 2) One tail for workset update, solution set update happens in an intermediate task
+ // 3) One tail for solution set update, workset update happens in an intermediate task
+
+ final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
+ final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
+
+ final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
+ final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
+
+ {
+ // get the vertex for the workset update
+ final TaskConfig worksetTailConfig;
+ AbstractJobVertex nextWorksetVertex = (AbstractJobVertex) this.vertices.get(nextWorksetNode);
+ if (nextWorksetVertex == null) {
+ // nextWorksetVertex is chained
+ TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
+ }
+ nextWorksetVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+ worksetTailConfig = taskInChain.getTaskConfig();
+ } else {
+ worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
+ }
+
+ // mark the node to perform workset updates
+ worksetTailConfig.setIsWorksetIteration();
+ worksetTailConfig.setIsWorksetUpdate();
+
+ if (hasWorksetTail) {
+ nextWorksetVertex.setInvokableClass(IterationTailPactTask.class);
+
+ worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
+ }
+ }
+ {
+ final TaskConfig solutionDeltaConfig;
+ AbstractJobVertex solutionDeltaVertex = (AbstractJobVertex) this.vertices.get(solutionDeltaNode);
+ if (solutionDeltaVertex == null) {
+ // last op is chained
+ TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
+ }
+ solutionDeltaVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+ solutionDeltaConfig = taskInChain.getTaskConfig();
+ } else {
+ solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
+ }
+
+ solutionDeltaConfig.setIsWorksetIteration();
+ solutionDeltaConfig.setIsSolutionSetUpdate();
+
+ if (hasSolutionSetTail) {
+ solutionDeltaVertex.setInvokableClass(IterationTailPactTask.class);
+
+ solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
+
+ // tell the head that it needs to wait for the solution set updates
+ headConfig.setWaitForSolutionSetUpdate();
+ }
+ else {
+ // no tail, intermediate update. must be immediate update
+ if (!iterNode.isImmediateSolutionSetUpdate()) {
+ throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates.");
+ }
+ solutionDeltaConfig.setIsSolutionSetUpdateWithoutReprobe();
+ }
+ }
+ }
+
+ // ------------------- register the aggregators -------------------
+ AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
+ Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
+
+ for (AggregatorWithName<?> agg : allAggregators) {
+ if (agg.getName().equals(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME)) {
+ throw new CompilerException("User defined aggregator used the same name as built-in workset " +
+ "termination check aggregator: " + WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
+ }
+ }
+
+ headConfig.addIterationAggregators(allAggregators);
+ syncConfig.addIterationAggregators(allAggregators);
+
+ String convAggName = aggs.getConvergenceCriterionAggregatorName();
+ ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
+
+ if (convCriterion != null || convAggName != null) {
+ throw new CompilerException("Error: Cannot use custom convergence criterion with workset iteration. Workset iterations have implicit convergence criterion where workset is empty.");
+ }
+
+ headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
+ syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
+ syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
+ }
+
+ private static String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
+ try {
+ if (wrapper.hasObject()) {
+ try {
+ return wrapper.getUserCodeObject().toString();
+ }
+ catch (Throwable t) {
+ return wrapper.getUserCodeClass().getName();
+ }
+ }
+ else {
+ return wrapper.getUserCodeClass().getName();
+ }
+ }
+ catch (Throwable t) {
+ return null;
+ }
+ }
+
+ // -------------------------------------------------------------------------------------
+ // Descriptors for tasks / configurations that are chained or merged with other tasks
+ // -------------------------------------------------------------------------------------
+
+ /**
+ * Utility class that describes a task in a sequence of chained tasks. Chained tasks are tasks that run
+ * together in one thread.
+ */
+ private static final class TaskInChain {
+
+ private final Class<? extends ChainedDriver<?, ?>> chainedTask;
+
+ private final TaskConfig taskConfig;
+
+ private final String taskName;
+
+ private AbstractJobVertex containingVertex;
+
+ TaskInChain(Class<? extends ChainedDriver<?, ?>> chainedTask, TaskConfig taskConfig,
+ String taskName) {
+ this.chainedTask = chainedTask;
+ this.taskConfig = taskConfig;
+ this.taskName = taskName;
+ }
+
+ public Class<? extends ChainedDriver<?, ?>> getChainedTask() {
+ return this.chainedTask;
+ }
+
+ public TaskConfig getTaskConfig() {
+ return this.taskConfig;
+ }
+
+ public String getTaskName() {
+ return this.taskName;
+ }
+
+ public AbstractJobVertex getContainingVertex() {
+ return this.containingVertex;
+ }
+
+ public void setContainingVertex(AbstractJobVertex containingVertex) {
+ this.containingVertex = containingVertex;
+ }
+ }
+
+ private static final class IterationDescriptor {
+
+ private final IterationPlanNode iterationNode;
+
+ private AbstractJobVertex headTask;
+
+ private TaskConfig headConfig;
+
+ private TaskConfig headFinalResultConfig;
+
+ private final int id;
+
+ public IterationDescriptor(IterationPlanNode iterationNode, int id) {
+ this.iterationNode = iterationNode;
+ this.id = id;
+ }
+
+ public IterationPlanNode getIterationNode() {
+ return iterationNode;
+ }
+
+ public void setHeadTask(AbstractJobVertex headTask, TaskConfig headConfig) {
+ this.headTask = headTask;
+ this.headFinalResultConfig = new TaskConfig(new Configuration());
+
+ // check if we already had a configuration, for example if the solution set was
+ if (this.headConfig != null) {
+ headConfig.getConfiguration().addAll(this.headConfig.getConfiguration());
+ }
+
+ this.headConfig = headConfig;
+ }
+
+ public AbstractJobVertex getHeadTask() {
+ return headTask;
+ }
+
+ public TaskConfig getHeadFinalResultConfig() {
+ return headFinalResultConfig;
+ }
+
+ public int getId() {
+ return this.id;
+ }
+ }
+}
[48/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/NepheleJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/NepheleJobGraphGenerator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/NepheleJobGraphGenerator.java
deleted file mode 100644
index ed6f5ca..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/NepheleJobGraphGenerator.java
+++ /dev/null
@@ -1,1581 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plantranslate;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.aggregators.AggregatorRegistry;
-import org.apache.flink.api.common.aggregators.AggregatorWithName;
-import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
-import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.operators.util.UserCodeWrapper;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
-import org.apache.flink.runtime.iterative.convergence.WorksetEmptyConvergenceCriterion;
-import org.apache.flink.runtime.iterative.task.IterationHeadPactTask;
-import org.apache.flink.runtime.iterative.task.IterationIntermediatePactTask;
-import org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask;
-import org.apache.flink.runtime.iterative.task.IterationTailPactTask;
-import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.InputFormatVertex;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.OutputFormatVertex;
-import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.runtime.operators.CoGroupDriver;
-import org.apache.flink.runtime.operators.CoGroupWithSolutionSetFirstDriver;
-import org.apache.flink.runtime.operators.CoGroupWithSolutionSetSecondDriver;
-import org.apache.flink.runtime.operators.DataSinkTask;
-import org.apache.flink.runtime.operators.DataSourceTask;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.JoinWithSolutionSetFirstDriver;
-import org.apache.flink.runtime.operators.JoinWithSolutionSetSecondDriver;
-import org.apache.flink.runtime.operators.MatchDriver;
-import org.apache.flink.runtime.operators.NoOpDriver;
-import org.apache.flink.runtime.operators.RegularPactTask;
-import org.apache.flink.runtime.operators.chaining.ChainedDriver;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.Visitor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * This component translates the optimizer's resulting plan a nephele job graph. The
- * translation is a one to one mapping. All decisions are made by the optimizer, this class
- * simply creates nephele data structures and descriptions corresponding to the optimizer's
- * result.
- * <p>
- * The basic method of operation is a top down traversal over the plan graph. On the way down, tasks are created
- * for the plan nodes, on the way back up, the nodes connect their predecessor.
- */
-public class NepheleJobGraphGenerator implements Visitor<PlanNode> {
-
- public static final String MERGE_ITERATION_AUX_TASKS_KEY = "compiler.merge-iteration-aux";
-
- private static final boolean mergeIterationAuxTasks = GlobalConfiguration.getBoolean(MERGE_ITERATION_AUX_TASKS_KEY, false);
-
-// private static final Logger LOG = LoggerFactory.getLogger(NepheleJobGraphGenerator.class);
-
- private static final TaskInChain ALREADY_VISITED_PLACEHOLDER = new TaskInChain(null, null, null);
-
- // ------------------------------------------------------------------------
-
- private Map<PlanNode, AbstractJobVertex> vertices; // a map from optimizer nodes to nephele vertices
-
- private Map<PlanNode, TaskInChain> chainedTasks; // a map from optimizer nodes to nephele vertices
-
- private Map<IterationPlanNode, IterationDescriptor> iterations;
-
- private List<TaskInChain> chainedTasksInSequence;
-
- private List<AbstractJobVertex> auxVertices; // auxiliary vertices which are added during job graph generation
-
- private final int defaultMaxFan;
-
- private final float defaultSortSpillingThreshold;
-
- private int iterationIdEnumerator = 1;
-
- private IterationPlanNode currentIteration; // the current the enclosing iteration
-
- private List<IterationPlanNode> iterationStack; // stack of enclosing iterations
-
- private SlotSharingGroup sharingGroup;
-
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new job graph generator that uses the default values for its resource configuration.
- */
- public NepheleJobGraphGenerator() {
- this.defaultMaxFan = ConfigConstants.DEFAULT_SPILLING_MAX_FAN;
- this.defaultSortSpillingThreshold = ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD;
- }
-
- public NepheleJobGraphGenerator(Configuration config) {
- this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY,
- ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
- this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
- ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
- }
-
- /**
- * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a
- * {@link org.apache.flink.runtime.jobgraph.JobGraph}.
- * This is an 1-to-1 mapping. No optimization whatsoever is applied.
- *
- * @param program
- * Optimized PACT plan that is translated into a JobGraph.
- * @return JobGraph generated from PACT plan.
- */
- public JobGraph compileJobGraph(OptimizedPlan program) {
- this.vertices = new HashMap<PlanNode, AbstractJobVertex>();
- this.chainedTasks = new HashMap<PlanNode, TaskInChain>();
- this.chainedTasksInSequence = new ArrayList<TaskInChain>();
- this.auxVertices = new ArrayList<AbstractJobVertex>();
- this.iterations = new HashMap<IterationPlanNode, IterationDescriptor>();
- this.iterationStack = new ArrayList<IterationPlanNode>();
-
- this.sharingGroup = new SlotSharingGroup();
-
- // generate Nephele job graph
- program.accept(this);
-
- // sanity check that we are not somehow in an iteration at the end
- if (this.currentIteration != null) {
- throw new CompilerException("The graph translation ended prematurely, leaving an unclosed iteration.");
- }
-
- // finalize the iterations
- for (IterationDescriptor iteration : this.iterations.values()) {
- if (iteration.getIterationNode() instanceof BulkIterationPlanNode) {
- finalizeBulkIteration(iteration);
- } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) {
- finalizeWorksetIteration(iteration);
- } else {
- throw new CompilerException();
- }
- }
-
- // now that the traversal is done, we have the chained tasks write their configs into their
- // parents' configurations
- for (int i = 0; i < this.chainedTasksInSequence.size(); i++) {
- TaskInChain tic = this.chainedTasksInSequence.get(i);
- TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
- t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
- }
-
- // create the jobgraph object
- JobGraph graph = new JobGraph(program.getJobName());
- graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());
- graph.setAllowQueuedScheduling(false);
-
- // add vertices to the graph
- for (AbstractJobVertex vertex : this.vertices.values()) {
- graph.addVertex(vertex);
- }
-
- for (AbstractJobVertex vertex : this.auxVertices) {
- graph.addVertex(vertex);
- vertex.setSlotSharingGroup(sharingGroup);
- }
-
- // add registered cache file into job configuration
- for (Entry<String, DistributedCacheEntry> e : program.getOriginalPactPlan().getCachedFiles()) {
- DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
- }
-
- try {
- InstantiationUtil.writeObjectToConfig(
- program.getOriginalPactPlan().getExecutionConfig(),
- graph.getJobConfiguration(),
- ExecutionConfig.CONFIG_KEY);
- } catch (IOException e) {
- throw new RuntimeException("Config object could not be written to Job Configuration: " + e);
- }
-
- // release all references again
- this.vertices = null;
- this.chainedTasks = null;
- this.chainedTasksInSequence = null;
- this.auxVertices = null;
- this.iterations = null;
- this.iterationStack = null;
-
- // return job graph
- return graph;
- }
-
- /**
- * This methods implements the pre-visiting during a depth-first traversal. It create the job vertex and
- * sets local strategy.
- *
- * @param node
- * The node that is currently processed.
- * @return True, if the visitor should descend to the node's children, false if not.
- * @see org.apache.flink.util.Visitor#preVisit(org.apache.flink.util.Visitable)
- */
- @Override
- public boolean preVisit(PlanNode node) {
- // check if we have visited this node before. in non-tree graphs, this happens
- if (this.vertices.containsKey(node) || this.chainedTasks.containsKey(node) || this.iterations.containsKey(node)) {
- // return false to prevent further descend
- return false;
- }
-
- // the vertex to be created for the current node
- final AbstractJobVertex vertex;
- try {
- if (node instanceof SinkPlanNode) {
- vertex = createDataSinkVertex((SinkPlanNode) node);
- }
- else if (node instanceof SourcePlanNode) {
- vertex = createDataSourceVertex((SourcePlanNode) node);
- }
- else if (node instanceof BulkIterationPlanNode) {
- BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
- // for the bulk iteration, we skip creating anything for now. we create the graph
- // for the step function in the post visit.
-
- // check that the root of the step function has the same DOP as the iteration.
- // because the tail must have the same DOP as the head, we can only merge the last
- // operator with the tail, if they have the same DOP. not merging is currently not
- // implemented
- PlanNode root = iterationNode.getRootOfStepFunction();
- if (root.getDegreeOfParallelism() != node.getDegreeOfParallelism())
- {
- throw new CompilerException("Error: The final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
-
- IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
- this.iterations.put(iterationNode, descr);
- vertex = null;
- }
- else if (node instanceof WorksetIterationPlanNode) {
- WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
-
- // we have the same constraints as for the bulk iteration
- PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
- PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
-
- if (nextWorkSet.getDegreeOfParallelism() != node.getDegreeOfParallelism())
- {
- throw new CompilerException("It is currently not supported that the final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
- if (solutionSetDelta.getDegreeOfParallelism() != node.getDegreeOfParallelism())
- {
- throw new CompilerException("It is currently not supported that the final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
-
- IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
- this.iterations.put(iterationNode, descr);
- vertex = null;
- }
- else if (node instanceof SingleInputPlanNode) {
- vertex = createSingleInputVertex((SingleInputPlanNode) node);
- }
- else if (node instanceof DualInputPlanNode) {
- vertex = createDualInputVertex((DualInputPlanNode) node);
- }
- else if (node instanceof NAryUnionPlanNode) {
- // skip the union for now
- vertex = null;
- }
- else if (node instanceof BulkPartialSolutionPlanNode) {
- // create a head node (or not, if it is merged into its successor)
- vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
- }
- else if (node instanceof SolutionSetPlanNode) {
- // this represents an access into the solution set index.
- // we do not create a vertex for the solution set here (we create the head at the workset place holder)
-
- // we adjust the joins / cogroups that go into the solution set here
- for (Channel c : node.getOutgoingChannels()) {
- DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
- AbstractJobVertex accessingVertex = this.vertices.get(target);
- TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
- int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
-
- // sanity checks
- if (inputNum == -1) {
- throw new CompilerException();
- }
-
- // adjust the driver
- if (conf.getDriver().equals(MatchDriver.class)) {
- conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
- }
- else if (conf.getDriver().equals(CoGroupDriver.class)) {
- conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
- }
- else {
- throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
- }
- }
-
- // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
- this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
-
- vertex = null;
- }
- else if (node instanceof WorksetPlanNode) {
- // create the iteration head here
- vertex = createWorksetIterationHead((WorksetPlanNode) node);
- }
- else {
- throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
- }
- }
- catch (Exception e) {
- throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
- }
-
- // check if a vertex was created, or if it was chained or skipped
- if (vertex != null) {
- // set degree of parallelism
- int pd = node.getDegreeOfParallelism();
- vertex.setParallelism(pd);
-
- vertex.setSlotSharingGroup(sharingGroup);
-
- // check whether this vertex is part of an iteration step function
- if (this.currentIteration != null) {
- // check that the task has the same DOP as the iteration as such
- PlanNode iterationNode = (PlanNode) this.currentIteration;
- if (iterationNode.getDegreeOfParallelism() < pd) {
- throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
- }
-
- // store the id of the iterations the step functions participate in
- IterationDescriptor descr = this.iterations.get(this.currentIteration);
- new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
- }
-
- // store in the map
- this.vertices.put(node, vertex);
- }
-
- // returning true causes deeper descend
- return true;
- }
-
- /**
- * This method implements the post-visit during the depth-first traversal. When the post visit happens,
- * all of the descendants have been processed, so this method connects all of the current node's
- * predecessors to the current node.
- *
- * @param node
- * The node currently processed during the post-visit.
- * @see org.apache.flink.util.Visitor#postVisit(org.apache.flink.util.Visitable) t
- */
- @Override
- public void postVisit(PlanNode node) {
- try {
- // --------- check special cases for which we handle post visit differently ----------
-
- // skip data source node (they have no inputs)
- // also, do nothing for union nodes, we connect them later when gathering the inputs for a task
- // solution sets have no input. the initial solution set input is connected when the iteration node is in its postVisit
- if (node instanceof SourcePlanNode || node instanceof NAryUnionPlanNode || node instanceof SolutionSetPlanNode) {
- return;
- }
-
- // check if we have an iteration. in that case, translate the step function now
- if (node instanceof IterationPlanNode) {
- // prevent nested iterations
- if (node.isOnDynamicPath()) {
- throw new CompilerException("Nested Iterations are not possible at the moment!");
- }
-
- // if we recursively go into an iteration (because the constant path of one iteration contains
- // another one), we push the current one onto the stack
- if (this.currentIteration != null) {
- this.iterationStack.add(this.currentIteration);
- }
-
- this.currentIteration = (IterationPlanNode) node;
- this.currentIteration.acceptForStepFunction(this);
-
- // pop the current iteration from the stack
- if (this.iterationStack.isEmpty()) {
- this.currentIteration = null;
- } else {
- this.currentIteration = this.iterationStack.remove(this.iterationStack.size() - 1);
- }
-
- // inputs for initial bulk partial solution or initial workset are already connected to the iteration head in the head's post visit.
- // connect the initial solution set now.
- if (node instanceof WorksetIterationPlanNode) {
- // connect the initial solution set
- WorksetIterationPlanNode wsNode = (WorksetIterationPlanNode) node;
- AbstractJobVertex headVertex = this.iterations.get(wsNode).getHeadTask();
- TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- int inputIndex = headConfig.getDriverStrategy().getNumInputs();
- headConfig.setIterationHeadSolutionSetInputIndex(inputIndex);
- translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
- }
-
- return;
- }
-
- final AbstractJobVertex targetVertex = this.vertices.get(node);
-
-
- // --------- Main Path: Translation of channels ----------
- //
- // There are two paths of translation: One for chained tasks (or merged tasks in general),
- // which do not have their own task vertex. The other for tasks that have their own vertex,
- // or are the primary task in a vertex (to which the others are chained).
-
- // check whether this node has its own task, or is merged with another one
- if (targetVertex == null) {
- // node's task is merged with another task. it is either chained, of a merged head vertex
- // from an iteration
- final TaskInChain chainedTask;
- if ((chainedTask = this.chainedTasks.get(node)) != null) {
- // Chained Task. Sanity check first...
- final Iterator<Channel> inConns = node.getInputs().iterator();
- if (!inConns.hasNext()) {
- throw new CompilerException("Bug: Found chained task with no input.");
- }
- final Channel inConn = inConns.next();
-
- if (inConns.hasNext()) {
- throw new CompilerException("Bug: Found a chained task with more than one input!");
- }
- if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
- throw new CompilerException("Bug: Found a chained task with an input local strategy.");
- }
- if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
- throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
- }
-
- AbstractJobVertex container = chainedTask.getContainingVertex();
-
- if (container == null) {
- final PlanNode sourceNode = inConn.getSource();
- container = this.vertices.get(sourceNode);
- if (container == null) {
- // predecessor is itself chained
- container = this.chainedTasks.get(sourceNode).getContainingVertex();
- if (container == null) {
- throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
- }
- } else {
- // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
- new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
- }
- chainedTask.setContainingVertex(container);
- }
-
- // add info about the input serializer type
- chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
-
- // update name of container task
- String containerTaskName = container.getName();
- if(containerTaskName.startsWith("CHAIN ")) {
- container.setName(containerTaskName+" -> "+chainedTask.getTaskName());
- } else {
- container.setName("CHAIN "+containerTaskName+" -> "+chainedTask.getTaskName());
- }
-
- this.chainedTasksInSequence.add(chainedTask);
- return;
- }
- else if (node instanceof BulkPartialSolutionPlanNode ||
- node instanceof WorksetPlanNode)
- {
- // merged iteration head task. the task that the head is merged with will take care of it
- return;
- } else {
- throw new CompilerException("Bug: Unrecognized merged task vertex.");
- }
- }
-
- // -------- Here, we translate non-chained tasks -------------
-
-
- if (this.currentIteration != null) {
- AbstractJobVertex head = this.iterations.get(this.currentIteration).getHeadTask();
- // the head may still be null if we descend into the static parts first
- if (head != null) {
- targetVertex.setStrictlyCoLocatedWith(head);
- }
- }
-
-
- // create the config that will contain all the description of the inputs
- final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
-
- // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
- // enclosing iteration node, because the inputs are the initial inputs to the iteration.
- final Iterator<Channel> inConns;
- if (node instanceof BulkPartialSolutionPlanNode) {
- inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
- // because the partial solution has its own vertex, is has only one (logical) input.
- // note this in the task configuration
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
- } else if (node instanceof WorksetPlanNode) {
- WorksetPlanNode wspn = (WorksetPlanNode) node;
- // input that is the initial workset
- inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
-
- // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
- // be input 0 and the solution set will be input 1
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
- targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
- } else {
- inConns = node.getInputs().iterator();
- }
- if (!inConns.hasNext()) {
- throw new CompilerException("Bug: Found a non-source task with no input.");
- }
-
- int inputIndex = 0;
- while (inConns.hasNext()) {
- Channel input = inConns.next();
- inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
- }
- // broadcast variables
- int broadcastInputIndex = 0;
- for (NamedChannel broadcastInput: node.getBroadcastInputs()) {
- int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true);
- targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
- targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex);
- broadcastInputIndex += broadcastInputIndexDelta;
- }
- } catch (Exception e) {
- throw new CompilerException(
- "An error occurred while translating the optimized plan to a nephele JobGraph: " + e.getMessage(), e);
- }
- }
-
- private int translateChannel(Channel input, int inputIndex, AbstractJobVertex targetVertex,
- TaskConfig targetVertexConfig, boolean isBroadcast) throws Exception
- {
- final PlanNode inputPlanNode = input.getSource();
- final Iterator<Channel> allInChannels;
-
- if (inputPlanNode instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) inputPlanNode).getListOfInputs().iterator();
- }
- else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
- if (this.vertices.get(inputPlanNode) == null) {
- // merged iteration head
- final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
- final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
-
- // check if the iteration's input is a union
- if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs().iterator();
- } else {
- allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
- }
-
- // also, set the index of the gate with the partial solution
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
- } else {
- // standalone iteration head
- allInChannels = Collections.singletonList(input).iterator();
- }
- } else if (inputPlanNode instanceof WorksetPlanNode) {
- if (this.vertices.get(inputPlanNode) == null) {
- // merged iteration head
- final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
- final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
-
- // check if the iteration's input is a union
- if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs().iterator();
- } else {
- allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
- }
-
- // also, set the index of the gate with the partial solution
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
- } else {
- // standalone iteration head
- allInChannels = Collections.singletonList(input).iterator();
- }
- } else if (inputPlanNode instanceof SolutionSetPlanNode) {
- // for now, skip connections with the solution set node, as this is a local index access (later to be parameterized here)
- // rather than a vertex connection
- return 0;
- } else {
- allInChannels = Collections.singletonList(input).iterator();
- }
-
- // check that the type serializer is consistent
- TypeSerializerFactory<?> typeSerFact = null;
-
- // accounting for channels on the dynamic path
- int numChannelsTotal = 0;
- int numChannelsDynamicPath = 0;
- int numDynamicSenderTasksTotal = 0;
-
-
- // expand the channel to all the union channels, in case there is a union operator at its source
- while (allInChannels.hasNext()) {
- final Channel inConn = allInChannels.next();
-
- // sanity check the common serializer
- if (typeSerFact == null) {
- typeSerFact = inConn.getSerializer();
- } else if (!typeSerFact.equals(inConn.getSerializer())) {
- throw new CompilerException("Conflicting types in union operator.");
- }
-
- final PlanNode sourceNode = inConn.getSource();
- AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
- TaskConfig sourceVertexConfig;
-
- if (sourceVertex == null) {
- // this predecessor is chained to another task or an iteration
- final TaskInChain chainedTask;
- final IterationDescriptor iteration;
- if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
- // push chained task
- if (chainedTask.getContainingVertex() == null) {
- throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
- }
- sourceVertex = chainedTask.getContainingVertex();
- sourceVertexConfig = chainedTask.getTaskConfig();
- } else if ((iteration = this.iterations.get(sourceNode)) != null) {
- // predecessor is an iteration
- sourceVertex = iteration.getHeadTask();
- sourceVertexConfig = iteration.getHeadFinalResultConfig();
- } else {
- throw new CompilerException("Bug: Could not resolve source node for a channel.");
- }
- } else {
- // predecessor is its own vertex
- sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
- }
- DistributionPattern pattern = connectJobVertices(
- inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
-
- // accounting on channels and senders
- numChannelsTotal++;
- if (inConn.isOnDynamicPath()) {
- numChannelsDynamicPath++;
- numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
- sourceVertex.getParallelism(), targetVertex.getParallelism());
- }
- }
-
- // for the iterations, check that the number of dynamic channels is the same as the number
- // of channels for this logical input. this condition is violated at the moment, if there
- // is a union between nodes on the static and nodes on the dynamic path
- if (numChannelsDynamicPath > 0 && numChannelsTotal != numChannelsDynamicPath) {
- throw new CompilerException("Error: It is currently not supported to union between dynamic and static path in an iteration.");
- }
- if (numDynamicSenderTasksTotal > 0) {
- if (isBroadcast) {
- targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
- } else {
- targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
- }
- }
-
- // the local strategy is added only once. in non-union case that is the actual edge,
- // in the union case, it is the edge between union and the target node
- addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
- return 1;
- }
-
- private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) {
- if (pattern == DistributionPattern.ALL_TO_ALL) {
- return numSenders;
- } else if (pattern == DistributionPattern.POINTWISE) {
- if (numSenders != numReceivers) {
- if (numReceivers == 1) {
- return numSenders;
- }
- else if (numSenders == 1) {
- return 1;
- }
- else {
- throw new CompilerException("Error: A changing degree of parallelism is currently " +
- "not supported between tasks within an iteration.");
- }
- } else {
- return 1;
- }
- } else {
- throw new CompilerException("Unknown distribution pattern for channels: " + pattern);
- }
- }
-
- // ------------------------------------------------------------------------
- // Methods for creating individual vertices
- // ------------------------------------------------------------------------
-
- private AbstractJobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
- final String taskName = node.getNodeName();
- final DriverStrategy ds = node.getDriverStrategy();
-
- // check, whether chaining is possible
- boolean chaining = false;
- {
- Channel inConn = node.getInput();
- PlanNode pred = inConn.getSource();
- chaining = ds.getPushChainDriverClass() != null &&
- !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged
- !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways
- !(pred instanceof WorksetPlanNode) && // workset merges anyways
- !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
- inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
- inConn.getLocalStrategy() == LocalStrategy.NONE &&
- pred.getOutgoingChannels().size() == 1 &&
- node.getDegreeOfParallelism() == pred.getDegreeOfParallelism() &&
- node.getBroadcastInputs().isEmpty();
-
- // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
- // in a tail
- if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
- node.getOutgoingChannels().size() > 0)
- {
- WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
- if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
- chaining = false;
- }
- }
- // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
- if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
- {
- BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
- if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
- chaining = false;
- }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
- wspn.getRootOfTerminationCriterion() == pred)) {
- chaining = false;
- }
- }
- }
-
- final AbstractJobVertex vertex;
- final TaskConfig config;
-
- if (chaining) {
- vertex = null;
- config = new TaskConfig(new Configuration());
- this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
- } else {
- // create task vertex
- vertex = new AbstractJobVertex(taskName);
- vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
-
- config = new TaskConfig(vertex.getConfiguration());
- config.setDriver(ds.getDriverClass());
- }
-
- // set user code
- config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
- config.setStubParameters(node.getPactContract().getParameters());
-
- // set the driver strategy
- config.setDriverStrategy(ds);
- for(int i=0;i<ds.getNumRequiredComparators();i++) {
- config.setDriverComparator(node.getComparator(i), i);
- }
- // assign memory, file-handles, etc.
- assignDriverResources(node, config);
- return vertex;
- }
-
- private AbstractJobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
- final String taskName = node.getNodeName();
- final DriverStrategy ds = node.getDriverStrategy();
- final AbstractJobVertex vertex = new AbstractJobVertex(taskName);
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
- vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
-
- // set user code
- config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
- config.setStubParameters(node.getPactContract().getParameters());
-
- // set the driver strategy
- config.setDriver(ds.getDriverClass());
- config.setDriverStrategy(ds);
- if (node.getComparator1() != null) {
- config.setDriverComparator(node.getComparator1(), 0);
- }
- if (node.getComparator2() != null) {
- config.setDriverComparator(node.getComparator2(), 1);
- }
- if (node.getPairComparator() != null) {
- config.setDriverPairComparator(node.getPairComparator());
- }
-
- // assign memory, file-handles, etc.
- assignDriverResources(node, config);
- return vertex;
- }
-
- private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
- final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
-
- vertex.setInvokableClass(DataSourceTask.class);
- vertex.setFormatDescription(getDescriptionForUserCode(node.getPactContract().getUserCodeWrapper()));
-
- // set user code
- config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
- config.setStubParameters(node.getPactContract().getParameters());
-
- config.setOutputSerializer(node.getSerializer());
- return vertex;
- }
-
- private AbstractJobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
- final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName());
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
-
- vertex.setInvokableClass(DataSinkTask.class);
- vertex.setFormatDescription(getDescriptionForUserCode(node.getPactContract().getUserCodeWrapper()));
-
- // set user code
- config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
- config.setStubParameters(node.getPactContract().getParameters());
-
- return vertex;
- }
-
- private AbstractJobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
- // get the bulk iteration that corresponds to this partial solution node
- final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
-
- // check whether we need an individual vertex for the partial solution, or whether we
- // attach ourselves to the vertex of the parent node. We can combine the head with a node of
- // the step function, if
- // 1) There is one parent that the partial solution connects to via a forward pattern and no
- // local strategy
- // 2) DOP and the number of subtasks per instance does not change
- // 3) That successor is not a union
- // 4) That successor is not itself the last node of the step function
- // 5) There is no local strategy on the edge for the initial partial solution, as
- // this translates to a local strategy that would only be executed in the first iteration
-
- final boolean merge;
- if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
- final Channel c = pspn.getOutgoingChannels().get(0);
- final PlanNode successor = c.getTarget();
- merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
- c.getLocalStrategy() == LocalStrategy.NONE &&
- c.getTempMode() == TempMode.NONE &&
- successor.getDegreeOfParallelism() == pspn.getDegreeOfParallelism() &&
- !(successor instanceof NAryUnionPlanNode) &&
- successor != iteration.getRootOfStepFunction() &&
- iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
- } else {
- merge = false;
- }
-
- // create or adopt the head vertex
- final AbstractJobVertex toReturn;
- final AbstractJobVertex headVertex;
- final TaskConfig headConfig;
- if (merge) {
- final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
- headVertex = (AbstractJobVertex) this.vertices.get(successor);
-
- if (headVertex == null) {
- throw new CompilerException(
- "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
- }
-
- // reset the vertex type to iteration head
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- toReturn = null;
- } else {
- // instantiate the head vertex and give it a no-op driver as the driver strategy.
- // everything else happens in the post visit, after the input (the initial partial solution)
- // is connected.
- headVertex = new AbstractJobVertex("PartialSolution ("+iteration.getNodeName()+")");
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- headConfig.setDriver(NoOpDriver.class);
- toReturn = headVertex;
- }
-
- // create the iteration descriptor and the iteration to it
- IterationDescriptor descr = this.iterations.get(iteration);
- if (descr == null) {
- throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
- }
- descr.setHeadTask(headVertex, headConfig);
-
- return toReturn;
- }
-
- private AbstractJobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
- // get the bulk iteration that corresponds to this partial solution node
- final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
-
- // check whether we need an individual vertex for the partial solution, or whether we
- // attach ourselves to the vertex of the parent node. We can combine the head with a node of
- // the step function, if
- // 1) There is one parent that the partial solution connects to via a forward pattern and no
- // local strategy
- // 2) DOP and the number of subtasks per instance does not change
- // 3) That successor is not a union
- // 4) That successor is not itself the last node of the step function
- // 5) There is no local strategy on the edge for the initial workset, as
- // this translates to a local strategy that would only be executed in the first superstep
-
- final boolean merge;
- if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
- final Channel c = wspn.getOutgoingChannels().get(0);
- final PlanNode successor = c.getTarget();
- merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
- c.getLocalStrategy() == LocalStrategy.NONE &&
- c.getTempMode() == TempMode.NONE &&
- successor.getDegreeOfParallelism() == wspn.getDegreeOfParallelism() &&
- !(successor instanceof NAryUnionPlanNode) &&
- successor != iteration.getNextWorkSetPlanNode() &&
- iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
- } else {
- merge = false;
- }
-
- // create or adopt the head vertex
- final AbstractJobVertex toReturn;
- final AbstractJobVertex headVertex;
- final TaskConfig headConfig;
- if (merge) {
- final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
- headVertex = (AbstractJobVertex) this.vertices.get(successor);
-
- if (headVertex == null) {
- throw new CompilerException(
- "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
- }
-
- // reset the vertex type to iteration head
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- toReturn = null;
- } else {
- // instantiate the head vertex and give it a no-op driver as the driver strategy.
- // everything else happens in the post visit, after the input (the initial partial solution)
- // is connected.
- headVertex = new AbstractJobVertex("IterationHead("+iteration.getNodeName()+")");
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- headConfig.setDriver(NoOpDriver.class);
- toReturn = headVertex;
- }
-
- headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
-
- // create the iteration descriptor and the iteration to it
- IterationDescriptor descr = this.iterations.get(iteration);
- if (descr == null) {
- throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
- }
- descr.setHeadTask(headVertex, headConfig);
-
- return toReturn;
- }
-
- private void assignDriverResources(PlanNode node, TaskConfig config) {
- final double relativeMem = node.getRelativeMemoryPerSubTask();
- if (relativeMem > 0) {
- config.setRelativeMemoryDriver(relativeMem);
- config.setFilehandlesDriver(this.defaultMaxFan);
- config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
- }
- }
-
- private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) {
- if (c.getRelativeMemoryLocalStrategy() > 0) {
- config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy());
- config.setFilehandlesInput(inputNum, this.defaultMaxFan);
- config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold);
- }
- }
-
- // ------------------------------------------------------------------------
- // Connecting Vertices
- // ------------------------------------------------------------------------
-
- /**
- * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
- * channel is then the channel into the union node, the local strategy channel the one from the union to the
- * actual target operator.
- *
- * @param channel
- * @param inputNumber
- * @param sourceVertex
- * @param sourceConfig
- * @param targetVertex
- * @param targetConfig
- * @param isBroadcast
- * @throws CompilerException
- */
- private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
- final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
- final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
- throws CompilerException
- {
- // ------------ connect the vertices to the job graph --------------
- final DistributionPattern distributionPattern;
-
- switch (channel.getShipStrategy()) {
- case FORWARD:
- distributionPattern = DistributionPattern.POINTWISE;
- break;
- case PARTITION_RANDOM:
- case BROADCAST:
- case PARTITION_HASH:
- case PARTITION_CUSTOM:
- case PARTITION_RANGE:
- case PARTITION_FORCED_REBALANCE:
- distributionPattern = DistributionPattern.ALL_TO_ALL;
- break;
- default:
- throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
- }
-
- final ResultPartitionType resultType;
-
- switch (channel.getDataExchangeMode()) {
-
- case PIPELINED:
- resultType = ResultPartitionType.PIPELINED;
- break;
-
- case BATCH:
- // BLOCKING results are currently not supported in closed loop iterations
- //
- // See https://issues.apache.org/jira/browse/FLINK-1713 for details
- resultType = channel.getSource().isOnDynamicPath()
- ? ResultPartitionType.PIPELINED
- : ResultPartitionType.BLOCKING;
- break;
-
- case PIPELINE_WITH_BATCH_FALLBACK:
- throw new UnsupportedOperationException("Data exchange mode " +
- channel.getDataExchangeMode() + " currently not supported.");
-
- default:
- throw new UnsupportedOperationException("Unknown data exchange mode.");
-
- }
-
- targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
-
- // -------------- configure the source task's ship strategy strategies in task config --------------
- final int outputIndex = sourceConfig.getNumOutputs();
- sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
- if (outputIndex == 0) {
- sourceConfig.setOutputSerializer(channel.getSerializer());
- }
- if (channel.getShipStrategyComparator() != null) {
- sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
- }
-
- if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
-
- final DataDistribution dataDistribution = channel.getDataDistribution();
- if (dataDistribution != null) {
- sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
- } else {
- throw new RuntimeException("Range partitioning requires data distribution");
- // TODO: inject code and configuration for automatic histogram generation
- }
- }
-
- if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
- if (channel.getPartitioner() != null) {
- sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
- } else {
- throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
- }
- }
-
- // ---------------- configure the receiver -------------------
- if (isBroadcast) {
- targetConfig.addBroadcastInputToGroup(inputNumber);
- } else {
- targetConfig.addInputToGroup(inputNumber);
- }
- return distributionPattern;
- }
-
- private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
- // serializer
- if (isBroadcastChannel) {
- config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
-
- if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
- throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
- } else {
- return;
- }
- } else {
- config.setInputSerializer(channel.getSerializer(), inputNum);
- }
-
- // local strategy
- if (channel.getLocalStrategy() != LocalStrategy.NONE) {
- config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
- if (channel.getLocalStrategyComparator() != null) {
- config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
- }
- }
-
- assignLocalStrategyResources(channel, config, inputNum);
-
- // materialization / caching
- if (channel.getTempMode() != null) {
- final TempMode tm = channel.getTempMode();
-
- boolean needsMemory = false;
- // Don't add a pipeline breaker if the data exchange is already blocking.
- if (tm.breaksPipeline() && channel.getDataExchangeMode() != DataExchangeMode.BATCH) {
- config.setInputAsynchronouslyMaterialized(inputNum, true);
- needsMemory = true;
- }
- if (tm.isCached()) {
- config.setInputCached(inputNum, true);
- needsMemory = true;
- }
-
- if (needsMemory) {
- // sanity check
- if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
- throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
- }
- config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
- }
- }
- }
-
- private void finalizeBulkIteration(IterationDescriptor descr) {
-
- final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
- final AbstractJobVertex headVertex = descr.getHeadTask();
- final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
-
- // ------------ finalize the head config with the final outputs and the sync gate ------------
- final int numStepFunctionOuts = headConfig.getNumOutputs();
- final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
-
- if (numStepFunctionOuts == 0) {
- throw new CompilerException("The iteration has no operation inside the step function.");
- }
-
- headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
- headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
- final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
- if (relativeMemForBackChannel <= 0) {
- throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
- }
- headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
-
- // --------------------------- create the sync task ---------------------------
- final AbstractJobVertex sync = new AbstractJobVertex("Sync(" + bulkNode.getNodeName() + ")");
- sync.setInvokableClass(IterationSynchronizationSinkTask.class);
- sync.setParallelism(1);
- this.auxVertices.add(sync);
-
- final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
- syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
-
- // set the number of iteration / convergence criterion for the sync
- final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
- if (maxNumIterations < 1) {
- throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
- }
- syncConfig.setNumberOfIterations(maxNumIterations);
-
- // connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
-
- // ----------------------------- create the iteration tail ------------------------------
-
- final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
- final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
- final TaskConfig tailConfig;
-
- AbstractJobVertex rootOfStepFunctionVertex = (AbstractJobVertex) this.vertices.get(rootOfStepFunction);
- if (rootOfStepFunctionVertex == null) {
- // last op is chained
- final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
- }
- rootOfStepFunctionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
-
- // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
- tailConfig = taskInChain.getTaskConfig();
- } else {
- tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
- }
-
- tailConfig.setIsWorksetUpdate();
-
- // No following termination criterion
- if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
-
- rootOfStepFunctionVertex.setInvokableClass(IterationTailPactTask.class);
-
- tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
- }
-
-
- // create the fake output task for termination criterion, if needed
- final TaskConfig tailConfigOfTerminationCriterion;
- // If we have a termination criterion and it is not an intermediate node
- if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
- AbstractJobVertex rootOfTerminationCriterionVertex = (AbstractJobVertex) this.vertices.get(rootOfTerminationCriterion);
-
-
- if (rootOfTerminationCriterionVertex == null) {
- // last op is chained
- final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
- }
- rootOfTerminationCriterionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
-
- // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
- tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
- } else {
- tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
- }
-
- rootOfTerminationCriterionVertex.setInvokableClass(IterationTailPactTask.class);
- // Hack
- tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
- tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
-
- // tell the head that it needs to wait for the solution set updates
- headConfig.setWaitForSolutionSetUpdate();
- }
-
- // ------------------- register the aggregators -------------------
- AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
- Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
-
- headConfig.addIterationAggregators(allAggregators);
- syncConfig.addIterationAggregators(allAggregators);
-
- String convAggName = aggs.getConvergenceCriterionAggregatorName();
- ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
-
- if (convCriterion != null || convAggName != null) {
- if (convCriterion == null) {
- throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
- }
- if (convAggName == null) {
- throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
- }
-
- syncConfig.setConvergenceCriterion(convAggName, convCriterion);
- }
- }
-
- private void finalizeWorksetIteration(IterationDescriptor descr) {
- final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
- final AbstractJobVertex headVertex = descr.getHeadTask();
- final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
-
- // ------------ finalize the head config with the final outputs and the sync gate ------------
- {
- final int numStepFunctionOuts = headConfig.getNumOutputs();
- final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
-
- if (numStepFunctionOuts == 0) {
- throw new CompilerException("The workset iteration has no operation on the workset inside the step function.");
- }
-
- headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
- headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
- final double relativeMemory = iterNode.getRelativeMemoryPerSubTask();
- if (relativeMemory <= 0) {
- throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
- }
-
- headConfig.setIsWorksetIteration();
- headConfig.setRelativeBackChannelMemory(relativeMemory / 2);
- headConfig.setRelativeSolutionSetMemory(relativeMemory / 2);
-
- // set the solution set serializer and comparator
- headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
- headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
- }
-
- // --------------------------- create the sync task ---------------------------
- final TaskConfig syncConfig;
- {
- final AbstractJobVertex sync = new AbstractJobVertex("Sync (" + iterNode.getNodeName() + ")");
- sync.setInvokableClass(IterationSynchronizationSinkTask.class);
- sync.setParallelism(1);
- this.auxVertices.add(sync);
-
- syncConfig = new TaskConfig(sync.getConfiguration());
- syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
-
- // set the number of iteration / convergence criterion for the sync
- final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
- if (maxNumIterations < 1) {
- throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
- }
- syncConfig.setNumberOfIterations(maxNumIterations);
-
- // connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
- }
-
- // ----------------------------- create the iteration tails -----------------------------
- // ----------------------- for next workset and solution set delta-----------------------
-
- {
- // we have three possible cases:
- // 1) Two tails, one for workset update, one for solution set update
- // 2) One tail for workset update, solution set update happens in an intermediate task
- // 3) One tail for solution set update, workset update happens in an intermediate task
-
- final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
- final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
-
- final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
- final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
-
- {
- // get the vertex for the workset update
- final TaskConfig worksetTailConfig;
- AbstractJobVertex nextWorksetVertex = (AbstractJobVertex) this.vertices.get(nextWorksetNode);
- if (nextWorksetVertex == null) {
- // nextWorksetVertex is chained
- TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
- }
- nextWorksetVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
- worksetTailConfig = taskInChain.getTaskConfig();
- } else {
- worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
- }
-
- // mark the node to perform workset updates
- worksetTailConfig.setIsWorksetIteration();
- worksetTailConfig.setIsWorksetUpdate();
-
- if (hasWorksetTail) {
- nextWorksetVertex.setInvokableClass(IterationTailPactTask.class);
-
- worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
- }
- }
- {
- final TaskConfig solutionDeltaConfig;
- AbstractJobVertex solutionDeltaVertex = (AbstractJobVertex) this.vertices.get(solutionDeltaNode);
- if (solutionDeltaVertex == null) {
- // last op is chained
- TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
- }
- solutionDeltaVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
- solutionDeltaConfig = taskInChain.getTaskConfig();
- } else {
- solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
- }
-
- solutionDeltaConfig.setIsWorksetIteration();
- solutionDeltaConfig.setIsSolutionSetUpdate();
-
- if (hasSolutionSetTail) {
- solutionDeltaVertex.setInvokableClass(IterationTailPactTask.class);
-
- solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
-
- // tell the head that it needs to wait for the solution set updates
- headConfig.setWaitForSolutionSetUpdate();
- }
- else {
- // no tail, intermediate update. must be immediate update
- if (!iterNode.isImmediateSolutionSetUpdate()) {
- throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates.");
- }
- solutionDeltaConfig.setIsSolutionSetUpdateWithoutReprobe();
- }
- }
- }
-
- // ------------------- register the aggregators -------------------
- AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
- Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
-
- for (AggregatorWithName<?> agg : allAggregators) {
- if (agg.getName().equals(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME)) {
- throw new CompilerException("User defined aggregator used the same name as built-in workset " +
- "termination check aggregator: " + WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
- }
- }
-
- headConfig.addIterationAggregators(allAggregators);
- syncConfig.addIterationAggregators(allAggregators);
-
- String convAggName = aggs.getConvergenceCriterionAggregatorName();
- ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
-
- if (convCriterion != null || convAggName != null) {
- throw new CompilerException("Error: Cannot use custom convergence criterion with workset iteration. Workset iterations have implicit convergence criterion where workset is empty.");
- }
-
- headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
- syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
- syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
- }
-
- private static String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
- try {
- if (wrapper.hasObject()) {
- try {
- return wrapper.getUserCodeObject().toString();
- }
- catch (Throwable t) {
- return wrapper.getUserCodeClass().getName();
- }
- }
- else {
- return wrapper.getUserCodeClass().getName();
- }
- }
- catch (Throwable t) {
- return null;
- }
- }
-
- // -------------------------------------------------------------------------------------
- // Descriptors for tasks / configurations that are chained or merged with other tasks
- // -------------------------------------------------------------------------------------
-
- /**
- * Utility class that describes a task in a sequence of chained tasks. Chained tasks are tasks that run
- * together in one thread.
- */
- private static final class TaskInChain {
-
- private final Class<? extends ChainedDriver<?, ?>> chainedTask;
-
- private final TaskConfig taskConfig;
-
- private final String taskName;
-
- private AbstractJobVertex containingVertex;
-
- TaskInChain(Class<? extends ChainedDriver<?, ?>> chainedTask, TaskConfig taskConfig,
- String taskName) {
- this.chainedTask = chainedTask;
- this.taskConfig = taskConfig;
- this.taskName = taskName;
- }
-
- public Class<? extends ChainedDriver<?, ?>> getChainedTask() {
- return this.chainedTask;
- }
-
- public TaskConfig getTaskConfig() {
- return this.taskConfig;
- }
-
- public String getTaskName() {
- return this.taskName;
- }
-
- public AbstractJobVertex getContainingVertex() {
- return this.containingVertex;
- }
-
- public void setContainingVertex(AbstractJobVertex containingVertex) {
- this.containingVertex = containingVertex;
- }
- }
-
- private static final class IterationDescriptor {
-
- private final IterationPlanNode iterationNode;
-
- private AbstractJobVertex headTask;
-
- private TaskConfig headConfig;
-
- private TaskConfig headFinalResultConfig;
-
- private final int id;
-
- public IterationDescriptor(IterationPlanNode iterationNode, int id) {
- this.iterationNode = iterationNode;
- this.id = id;
- }
-
- public IterationPlanNode getIterationNode() {
- return iterationNode;
- }
-
- public void setHeadTask(AbstractJobVertex headTask, TaskConfig headConfig) {
- this.headTask = headTask;
- this.headFinalResultConfig = new TaskConfig(new Configuration());
-
- // check if we already had a configuration, for example if the solution set was
- if (this.headConfig != null) {
- headConfig.getConfiguration().addAll(this.headConfig.getConfiguration());
- }
-
- this.headConfig = headConfig;
- }
-
- public AbstractJobVertex getHeadTask() {
- return headTask;
- }
-
- public TaskConfig getHeadFinalResultConfig() {
- return headFinalResultConfig;
- }
-
- public int getId() {
- return this.id;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
index 3fd7457..2d8377e 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
@@ -89,7 +89,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
}
catch (ConflictingFieldTypeInfoException e) {
throw new CompilerPostPassException("Conflicting type infomation for the data sink '" +
- sn.getSinkNode().getPactContract().getName() + "'.");
+ sn.getSinkNode().getOperator().getName() + "'.");
}
// descend to the input channel
@@ -98,7 +98,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
}
catch (MissingFieldTypeInfoException ex) {
throw new CompilerPostPassException("Missing type infomation for the channel that inputs to the data sink '" +
- sn.getSinkNode().getPactContract().getName() + "'.");
+ sn.getSinkNode().getOperator().getName() + "'.");
}
}
else if (node instanceof SourcePlanNode) {
@@ -122,7 +122,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
// add the parent schema to the schema
if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
+ addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
}
// check whether all outgoing channels have not yet contributed. come back later if not.
@@ -168,7 +168,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
// take the schema from the partial solution node and add its fields to the iteration result schema.
// input and output schema need to be identical, so this is essentially a sanity check
- addSchemaToSchema(pss, schema, iterationNode.getPactContract().getName());
+ addSchemaToSchema(pss, schema, iterationNode.getProgramOperator().getName());
// set the serializer
if (createUtilities) {
@@ -180,7 +180,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(schema, iterationNode.getInput(), createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
- + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
+ + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
e.getFieldNumber());
}
}
@@ -199,7 +199,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
// add the parent schema to the schema (which refers to the solution set schema)
if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
+ addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
}
// check whether all outgoing channels have not yet contributed. come back later if not.
@@ -242,7 +242,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
}
} catch (ConflictingFieldTypeInfoException e) {
throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + iterationNode.getPactContract().getName() + "'. Contradicting types between the " +
+ + " in node '" + iterationNode.getProgramOperator().getName() + "'. Contradicting types between the " +
"result of the iteration and the solution set schema: " + e.getPreviousType() +
" and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
}
@@ -256,7 +256,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
} catch (MissingFieldTypeInfoException ex) {
throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
- optNode.getPactContract().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
+ optNode.getOperator().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
}
}
@@ -266,7 +266,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
} catch (MissingFieldTypeInfoException ex) {
throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
- + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
+ + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
ex.getFieldNumber());
}
}
@@ -298,7 +298,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
try {
getSingleInputNodeSchema(sn, schema);
} catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
+ throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
}
if (createUtilities) {
@@ -308,7 +308,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
sn.setComparator(createComparator(sn.getKeys(i), sn.getSortOrders(i), schema),i);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getPactContract().getName() + "'. Missing type information for key field " +
+ optNode.getOperator().getName() + "'. Missing type information for key field " +
e.getFieldNumber());
}
}
@@ -319,7 +319,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(schema, sn.getInput(), createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" +
- optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
// don't forget the broadcast inputs
@@ -328,7 +328,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(createEmptySchema(), c, createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
- optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
}
}
@@ -367,7 +367,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
try {
getDualInputNodeSchema(dn, schema1, schema2);
} catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
+ throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
}
// parameterize the node's driver strategy
@@ -379,7 +379,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
// set the pair comparator
@@ -388,7 +388,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
dn.getSortOrders(), schema1, schema2));
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
}
@@ -399,13 +399,13 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(schema1, dn.getInput1(), createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
- + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
try {
propagateToChannel(schema2, dn.getInput2(), createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
- + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
// don't forget the broadcast inputs
@@ -414,7 +414,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
propagateToChannel(createEmptySchema(), c, createUtilities);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
- optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
}
}
}
@@ -446,7 +446,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
schema = (T) psn.postPassHelper;
}
name = "partial solution of bulk iteration '" +
- psn.getPartialSolutionNode().getIterationNode().getPactContract().getName() + "'";
+ psn.getPartialSolutionNode().getIterationNode().getOperator().getName() + "'";
}
else if (node instanceof SolutionSetPlanNode) {
SolutionSetPlanNode ssn = (SolutionSetPlanNode) node;
@@ -457,7 +457,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
schema = (T) ssn.postPassHelper;
}
name = "solution set of workset iteration '" +
- ssn.getSolutionSetNode().getIterationNode().getPactContract().getName() + "'";
+ ssn.getSolutionSetNode().getIterationNode().getOperator().getName() + "'";
}
else if (node instanceof WorksetPlanNode) {
WorksetPlanNode wsn = (WorksetPlanNode) node;
@@ -468,7 +468,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
schema = (T) wsn.postPassHelper;
}
name = "workset of workset iteration '" +
- wsn.getWorksetNode().getIterationNode().getPactContract().getName() + "'";
+ wsn.getWorksetNode().getIterationNode().getOperator().getName() + "'";
} else {
throw new CompilerException();
}
@@ -531,7 +531,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
}
} catch (ConflictingFieldTypeInfoException e) {
throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + optNode.getPactContract().getName() + "' propagated from successor node. " +
+ + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " +
"Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() +
". Most probable cause: Invalid constant field annotations.");
}
@@ -550,7 +550,7 @@ public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> im
return createSerializer(schema);
} catch (MissingFieldTypeInfoException e) {
throw new CompilerPostPassException("Missing type information while creating serializer for '" +
- node.getPactContract().getName() + "'.");
+ node.getProgramOperator().getName() + "'.");
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
index 55d3269..5fdf3dd 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
@@ -114,7 +114,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
traverseChannel(addMapper.getInput());
}
- BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getPactContract();
+ BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
// set the serializer
iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
@@ -133,7 +133,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
}
- DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getPactContract();
+ DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
// set the serializers and comparators for the workset iteration
iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
@@ -152,10 +152,10 @@ public class JavaApiPostPass implements OptimizerPostPass {
else if (node instanceof SingleInputPlanNode) {
SingleInputPlanNode sn = (SingleInputPlanNode) node;
- if (!(sn.getOptimizerNode().getPactContract() instanceof SingleInputOperator)) {
+ if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
// Special case for delta iterations
- if(sn.getOptimizerNode().getPactContract() instanceof NoOpUnaryUdfOp) {
+ if(sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
traverseChannel(sn.getInput());
return;
} else {
@@ -163,7 +163,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
}
}
- SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getPactContract();
+ SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
// parameterize the node's driver strategy
for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
@@ -181,11 +181,11 @@ public class JavaApiPostPass implements OptimizerPostPass {
else if (node instanceof DualInputPlanNode) {
DualInputPlanNode dn = (DualInputPlanNode) node;
- if (!(dn.getOptimizerNode().getPactContract() instanceof DualInputOperator)) {
+ if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
throw new RuntimeException("Wrong operator type found in post pass.");
}
- DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getPactContract();
+ DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
// parameterize the node's driver strategy
if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
@@ -229,7 +229,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
private void traverseChannel(Channel channel) {
PlanNode source = channel.getSource();
- Operator<?> javaOp = source.getPactContract();
+ Operator<?> javaOp = source.getProgramOperator();
// if (!(javaOp instanceof BulkIteration) && !(javaOp instanceof JavaPlanNode)) {
// throw new RuntimeException("Wrong operator type found in post pass: " + javaOp);
@@ -271,7 +271,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
@SuppressWarnings("unchecked")
private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) {
- Operator<?> op = node.getOptimizerNode().getPactContract();
+ Operator<?> op = node.getOptimizerNode().getOperator();
if (op instanceof GenericDataSourceBase) {
return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
index dae7a2e..8a2d006 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
@@ -54,7 +54,7 @@ public class RecordModelPostPass extends GenericFlatTypePostPass<Class<? extends
@Override
protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException {
- GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getPactContract();
+ GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getOperator();
Ordering partitioning = sink.getPartitionOrdering();
Ordering sorting = sink.getLocalOrder();
@@ -76,7 +76,7 @@ public class RecordModelPostPass extends GenericFlatTypePostPass<Class<? extends
throws CompilerPostPassException, ConflictingFieldTypeInfoException
{
// check that we got the right types
- SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getPactContract();
+ SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getOperator();
if (! (contract instanceof RecordOperator)) {
throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation.");
}
@@ -103,7 +103,7 @@ public class RecordModelPostPass extends GenericFlatTypePostPass<Class<? extends
throws CompilerPostPassException, ConflictingFieldTypeInfoException
{
// add the nodes local information. this automatically consistency checks
- DualInputOperator<?, ?, ?, ?> contract = node.getTwoInputNode().getPactContract();
+ DualInputOperator<?, ?, ?, ?> contract = node.getTwoInputNode().getOperator();
if (! (contract instanceof RecordOperator)) {
throw new CompilerPostPassException("Error: Operator is not a Pact Record based contract. Wrong compiler invokation.");
}
[03/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
new file mode 100644
index 0000000..5175d8c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataexchange;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
+import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * This test verifies that the optimizer assigns the correct
+ * data exchange mode to a simple forward / shuffle plan.
+ *
+ * <pre>
+ * (source) -> (map) -> (filter) -> (groupBy / reduce)
+ * </pre>
+ */
+@SuppressWarnings("serial")
+public class DataExchangeModeForwardTest extends CompilerTestBase {
+
+
+ @Test
+ public void testPipelinedForced() {
+ // PIPELINED_FORCED should result in pipelining all the way
+ verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testPipelined() {
+ // PIPELINED should result in pipelining all the way
+ verifySimpleForwardPlan(ExecutionMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testBatch() {
+ // BATCH should result in batching the shuffle all the way
+ verifySimpleForwardPlan(ExecutionMode.BATCH,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testBatchForced() {
+ // BATCH_FORCED should result in batching all the way
+ verifySimpleForwardPlan(ExecutionMode.BATCH_FORCED,
+ DataExchangeMode.BATCH, DataExchangeMode.BATCH,
+ DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
+ DataExchangeMode.BATCH, DataExchangeMode.BATCH);
+ }
+
+ private void verifySimpleForwardPlan(ExecutionMode execMode,
+ DataExchangeMode toMap,
+ DataExchangeMode toFilter,
+ DataExchangeMode toKeyExtractor,
+ DataExchangeMode toCombiner,
+ DataExchangeMode toReduce,
+ DataExchangeMode toSink)
+ {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.getConfig().setExecutionMode(execMode);
+
+ DataSet<String> dataSet = env.readTextFile("/never/accessed");
+ dataSet
+ .map(new MapFunction<String, Integer>() {
+ @Override
+ public Integer map(String value) {
+ return 0;
+ }
+ })
+ .filter(new FilterFunction<Integer>() {
+ @Override
+ public boolean filter(Integer value) {
+ return false;
+ }
+ })
+ .groupBy(new IdentityKeyExtractor<Integer>())
+ .reduceGroup(new Top1GroupReducer<Integer>())
+ .output(new DiscardingOutputFormat<Integer>());
+
+ OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
+ SinkPlanNode sinkNode = optPlan.getDataSinks().iterator().next();
+
+ SingleInputPlanNode reduceNode = (SingleInputPlanNode) sinkNode.getPredecessor();
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getPredecessor();
+ SingleInputPlanNode keyExtractorNode = (SingleInputPlanNode) combineNode.getPredecessor();
+
+ SingleInputPlanNode filterNode = (SingleInputPlanNode) keyExtractorNode.getPredecessor();
+ SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
+
+ assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
+ assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
+ assertEquals(toKeyExtractor, keyExtractorNode.getInput().getDataExchangeMode());
+ assertEquals(toCombiner, combineNode.getInput().getDataExchangeMode());
+ assertEquals(toReduce, reduceNode.getInput().getDataExchangeMode());
+ assertEquals(toSink, sinkNode.getInput().getDataExchangeMode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
new file mode 100644
index 0000000..6b2691a
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataexchange;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.junit.Test;
+
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * This test checks the correct assignment of the DataExchangeMode to
+ * connections for programs that branch, but do not re-join the branches.
+ *
+ * <pre>
+ * /---> (filter) -> (sink)
+ * /
+ * /
+ * (source) -> (map) -----------------\
+ * \ (join) -> (sink)
+ * \ (source) --/
+ * \
+ * \
+ * \-> (sink)
+ * </pre>
+ */
+@SuppressWarnings({"serial", "unchecked"})
+public class DataExchangeModeOpenBranchingTest extends CompilerTestBase {
+
+ @Test
+ public void testPipelinedForced() {
+ // PIPELINED_FORCED should result in pipelining all the way
+ verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testPipelined() {
+ // PIPELINED should result in pipelining all the way
+ verifyBranchigPlan(ExecutionMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testBatch() {
+ // BATCH should result in batching the shuffle all the way
+ verifyBranchigPlan(ExecutionMode.BATCH,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
+ DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testBatchForced() {
+ // BATCH_FORCED should result in batching all the way
+ verifyBranchigPlan(ExecutionMode.BATCH_FORCED,
+ DataExchangeMode.BATCH, DataExchangeMode.BATCH,
+ DataExchangeMode.BATCH, DataExchangeMode.BATCH,
+ DataExchangeMode.BATCH, DataExchangeMode.BATCH,
+ DataExchangeMode.BATCH);
+ }
+
+ private void verifyBranchigPlan(ExecutionMode execMode,
+ DataExchangeMode toMap,
+ DataExchangeMode toFilter,
+ DataExchangeMode toFilterSink,
+ DataExchangeMode toJoin1,
+ DataExchangeMode toJoin2,
+ DataExchangeMode toJoinSink,
+ DataExchangeMode toDirectSink)
+ {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.getConfig().setExecutionMode(execMode);
+
+ DataSet<Tuple2<Long, Long>> data = env.generateSequence(1, 100000)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value) {
+ return new Tuple2<Long, Long>(value, value);
+ }
+ });
+
+ // output 1
+ data
+ .filter(new FilterFunction<Tuple2<Long, Long>>() {
+ @Override
+ public boolean filter(Tuple2<Long, Long> value) {
+ return false;
+ }
+ })
+ .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("sink1");
+
+ // output 2 does a join before a join
+ data
+ .join(env.fromElements(new Tuple2<Long, Long>(1L, 2L)))
+ .where(1)
+ .equalTo(0)
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()).name("sink2");
+
+ // output 3 is direct
+ data
+ .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("sink3");
+
+ OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
+
+ SinkPlanNode filterSink = findSink(optPlan.getDataSinks(), "sink1");
+ SinkPlanNode joinSink = findSink(optPlan.getDataSinks(), "sink2");
+ SinkPlanNode directSink = findSink(optPlan.getDataSinks(), "sink3");
+
+ SingleInputPlanNode filterNode = (SingleInputPlanNode) filterSink.getPredecessor();
+ SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
+
+ DualInputPlanNode joinNode = (DualInputPlanNode) joinSink.getPredecessor();
+ assertEquals(mapNode, joinNode.getInput1().getSource());
+
+ assertEquals(mapNode, directSink.getPredecessor());
+
+ assertEquals(toFilterSink, filterSink.getInput().getDataExchangeMode());
+ assertEquals(toJoinSink, joinSink.getInput().getDataExchangeMode());
+ assertEquals(toDirectSink, directSink.getInput().getDataExchangeMode());
+
+ assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
+ assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
+
+ assertEquals(toJoin1, joinNode.getInput1().getDataExchangeMode());
+ assertEquals(toJoin2, joinNode.getInput2().getDataExchangeMode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
+ for (SinkPlanNode node : collection) {
+ String nodeName = node.getOptimizerNode().getOperator().getName();
+ if (nodeName != null && nodeName.equals(name)) {
+ return node;
+ }
+ }
+
+ throw new IllegalArgumentException("No node with that name was found.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
new file mode 100644
index 0000000..1a14be5
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataexchange;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dag.SinkJoiner;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityFlatMapper;
+import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
+import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
+import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
+import org.apache.flink.optimizer.traversals.BranchesVisitor;
+import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
+import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
+import org.junit.Test;
+
+import java.util.Iterator;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * This test checks whether connections are correctly marked as pipelined breaking.
+ */
+@SuppressWarnings("serial")
+public class PipelineBreakingTest {
+
+ /**
+ * Tests that no pipeline breakers are inserted into a simple forward
+ * pipeline.
+ *
+ * <pre>
+ * (source) -> (map) -> (filter) -> (groupBy / reduce)
+ * </pre>
+ */
+ @Test
+ public void testSimpleForwardPlan() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> dataSet = env.readTextFile("/never/accessed");
+ dataSet
+ .map(new MapFunction<String, Integer>() {
+ @Override
+ public Integer map(String value) {
+ return 0;
+ }
+ })
+ .filter(new FilterFunction<Integer>() {
+ @Override
+ public boolean filter(Integer value) {
+ return false;
+ }
+ })
+ .groupBy(new IdentityKeyExtractor<Integer>())
+ .reduceGroup(new Top1GroupReducer<Integer>())
+ .output(new DiscardingOutputFormat<Integer>());
+
+ DataSinkNode sinkNode = convertPlan(env.createProgramPlan()).get(0);
+
+ SingleInputNode reduceNode = (SingleInputNode) sinkNode.getPredecessorNode();
+ SingleInputNode keyExtractorNode = (SingleInputNode) reduceNode.getPredecessorNode();
+
+ SingleInputNode filterNode = (SingleInputNode) keyExtractorNode.getPredecessorNode();
+ SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode();
+
+ assertFalse(sinkNode.getInputConnection().isBreakingPipeline());
+ assertFalse(reduceNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(keyExtractorNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(filterNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Tests that branching plans, where the branches are not re-joined,
+ * do not place pipeline breakers.
+ *
+ * <pre>
+ * /---> (filter) -> (sink)
+ * /
+ * /
+ * (source) -> (map) -----------------\
+ * \ (join) -> (sink)
+ * \ (source) --/
+ * \
+ * \
+ * \-> (sink)
+ * </pre>
+ */
+ @Test
+ public void testBranchingPlanNotReJoined() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Integer> data = env.readTextFile("/never/accessed")
+ .map(new MapFunction<String, Integer>() {
+ @Override
+ public Integer map(String value) {
+ return 0;
+ }
+ });
+
+ // output 1
+ data
+ .filter(new FilterFunction<Integer>() {
+ @Override
+ public boolean filter(Integer value) {
+ return false;
+ }
+ })
+ .output(new DiscardingOutputFormat<Integer>());
+
+ // output 2 does a join before a join
+ data
+ .join(env.fromElements(1, 2, 3, 4))
+ .where(new IdentityKeyExtractor<Integer>())
+ .equalTo(new IdentityKeyExtractor<Integer>())
+ .output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
+
+ // output 3 is direct
+ data
+ .output(new DiscardingOutputFormat<Integer>());
+
+ List<DataSinkNode> sinks = convertPlan(env.createProgramPlan());
+
+ // gather the optimizer DAG nodes
+
+ DataSinkNode sinkAfterFilter = sinks.get(0);
+ DataSinkNode sinkAfterJoin = sinks.get(1);
+ DataSinkNode sinkDirect = sinks.get(2);
+
+ SingleInputNode filterNode = (SingleInputNode) sinkAfterFilter.getPredecessorNode();
+ SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode();
+
+ TwoInputNode joinNode = (TwoInputNode) sinkAfterJoin.getPredecessorNode();
+ SingleInputNode joinInput = (SingleInputNode) joinNode.getSecondPredecessorNode();
+
+ // verify the non-pipeline breaking status
+
+ assertFalse(sinkAfterFilter.getInputConnection().isBreakingPipeline());
+ assertFalse(sinkAfterJoin.getInputConnection().isBreakingPipeline());
+ assertFalse(sinkDirect.getInputConnection().isBreakingPipeline());
+
+ assertFalse(filterNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
+
+ assertFalse(joinNode.getFirstIncomingConnection().isBreakingPipeline());
+ assertFalse(joinNode.getSecondIncomingConnection().isBreakingPipeline());
+ assertFalse(joinInput.getIncomingConnection().isBreakingPipeline());
+
+ // some other sanity checks on the plan construction (cannot hurt)
+
+ assertEquals(mapNode, ((SingleInputNode) joinNode.getFirstPredecessorNode()).getPredecessorNode());
+ assertEquals(mapNode, sinkDirect.getPredecessorNode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Tests that branches that are re-joined have place pipeline breakers.
+ *
+ * <pre>
+ * /-> (sink)
+ * /
+ * /-> (reduce) -+ /-> (flatmap) -> (sink)
+ * / \ /
+ * (source) -> (map) - (join) -+-----\
+ * \ / \
+ * \-> (filter) -+ \
+ * \ (co group) -> (sink)
+ * \ /
+ * \-> (reduce) - /
+ * </pre>
+ */
+ @Test
+ public void testReJoinedBranches() {
+ try {
+ // build a test program
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> data = env.fromElements(33L, 44L)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value) {
+ return new Tuple2<Long, Long>(value, value);
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> reduced = data.groupBy(0).reduce(new SelectOneReducer<Tuple2<Long, Long>>());
+ reduced.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> filtered = data.filter(new FilterFunction<Tuple2<Long, Long>>() {
+ @Override
+ public boolean filter(Tuple2<Long, Long> value) throws Exception {
+ return false;
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> joined = reduced.join(filtered)
+ .where(1).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ joined.flatMap(new IdentityFlatMapper<Tuple2<Long, Long>>())
+ .output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+
+ joined.coGroup(filtered.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()))
+ .where(0).equalTo(0)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
+
+ List<DataSinkNode> sinks = convertPlan(env.createProgramPlan());
+
+ // gather the optimizer DAG nodes
+
+ DataSinkNode sinkAfterReduce = sinks.get(0);
+ DataSinkNode sinkAfterFlatMap = sinks.get(1);
+ DataSinkNode sinkAfterCoGroup = sinks.get(2);
+
+ SingleInputNode reduceNode = (SingleInputNode) sinkAfterReduce.getPredecessorNode();
+ SingleInputNode mapNode = (SingleInputNode) reduceNode.getPredecessorNode();
+
+ SingleInputNode flatMapNode = (SingleInputNode) sinkAfterFlatMap.getPredecessorNode();
+ TwoInputNode joinNode = (TwoInputNode) flatMapNode.getPredecessorNode();
+ SingleInputNode filterNode = (SingleInputNode) joinNode.getSecondPredecessorNode();
+
+ TwoInputNode coGroupNode = (TwoInputNode) sinkAfterCoGroup.getPredecessorNode();
+ SingleInputNode otherReduceNode = (SingleInputNode) coGroupNode.getSecondPredecessorNode();
+
+ // test sanity checks (that we constructed the DAG correctly)
+
+ assertEquals(reduceNode, joinNode.getFirstPredecessorNode());
+ assertEquals(mapNode, filterNode.getPredecessorNode());
+ assertEquals(joinNode, coGroupNode.getFirstPredecessorNode());
+ assertEquals(filterNode, otherReduceNode.getPredecessorNode());
+
+ // verify the pipeline breaking status
+
+ assertFalse(sinkAfterReduce.getInputConnection().isBreakingPipeline());
+ assertFalse(sinkAfterFlatMap.getInputConnection().isBreakingPipeline());
+ assertFalse(sinkAfterCoGroup.getInputConnection().isBreakingPipeline());
+
+ assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(flatMapNode.getIncomingConnection().isBreakingPipeline());
+ assertFalse(joinNode.getFirstIncomingConnection().isBreakingPipeline());
+ assertFalse(coGroupNode.getFirstIncomingConnection().isBreakingPipeline());
+ assertFalse(coGroupNode.getSecondIncomingConnection().isBreakingPipeline());
+
+ // these should be pipeline breakers
+ assertTrue(reduceNode.getIncomingConnection().isBreakingPipeline());
+ assertTrue(filterNode.getIncomingConnection().isBreakingPipeline());
+ assertTrue(otherReduceNode.getIncomingConnection().isBreakingPipeline());
+ assertTrue(joinNode.getSecondIncomingConnection().isBreakingPipeline());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static List<DataSinkNode> convertPlan(Plan p) {
+ GraphCreatingVisitor dagCreator =
+ new GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
+
+ // create the DAG
+ p.accept(dagCreator);
+ List<DataSinkNode> sinks = dagCreator.getSinks();
+
+ // build a single root and run the branch tracking logic
+ OptimizerNode rootNode;
+ if (sinks.size() == 1) {
+ rootNode = sinks.get(0);
+ }
+ else {
+ Iterator<DataSinkNode> iter = sinks.iterator();
+ rootNode = iter.next();
+
+ while (iter.hasNext()) {
+ rootNode = new SinkJoiner(rootNode, iter.next());
+ }
+ }
+ rootNode.accept(new IdAndEstimatesVisitor(null));
+ rootNode.accept(new BranchesVisitor());
+
+ return sinks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
new file mode 100644
index 0000000..3e32905
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.StringValue;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Set;
+
+public class GlobalPropertiesFilteringTest {
+
+ private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
+ new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ @Test
+ public void testAllErased1() {
+
+ SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1));
+ gprops.addUniqueFieldCombination(new FieldSet(3, 4));
+ gprops.addUniqueFieldCombination(new FieldSet(5, 6));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ assertNull(result.getPartitioningOrdering());
+ assertNull(result.getUniqueFieldCombination());
+ }
+
+ @Test
+ public void testAllErased2() {
+
+ SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[]{"2"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1));
+ gprops.addUniqueFieldCombination(new FieldSet(3, 4));
+ gprops.addUniqueFieldCombination(new FieldSet(5, 6));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ assertNull(result.getPartitioningOrdering());
+ assertNull(result.getUniqueFieldCombination());
+ }
+
+ @Test
+ public void testHashPartitioningPreserved1() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertTrue(pFields.contains(0));
+ assertTrue(pFields.contains(1));
+ assertTrue(pFields.contains(4));
+ }
+
+ @Test
+ public void testHashPartitioningPreserved2() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertTrue(pFields.contains(1));
+ assertTrue(pFields.contains(2));
+ assertTrue(pFields.contains(3));
+ }
+
+ @Test
+ public void testHashPartitioningErased() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ }
+
+ @Test
+ public void testAnyPartitioningPreserved1() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setAnyPartitioning(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertTrue(pFields.contains(0));
+ assertTrue(pFields.contains(1));
+ assertTrue(pFields.contains(4));
+ }
+
+ @Test
+ public void testAnyPartitioningPreserved2() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setAnyPartitioning(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertTrue(pFields.contains(1));
+ assertTrue(pFields.contains(2));
+ assertTrue(pFields.contains(3));
+ }
+
+ @Test
+ public void testAnyPartitioningErased() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setAnyPartitioning(new FieldList(0, 1, 4));
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ }
+
+ @Test
+ public void testCustomPartitioningPreserved1() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
+ gprops.setCustomPartitioned(new FieldList(0, 4), myP);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(2, pFields.size());
+ assertTrue(pFields.contains(0));
+ assertTrue(pFields.contains(4));
+ assertEquals(myP, result.getCustomPartitioner());
+ }
+
+ @Test
+ public void testCustomPartitioningPreserved2() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
+ gprops.setCustomPartitioned(new FieldList(0, 4), myP);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(2, pFields.size());
+ assertTrue(pFields.contains(1));
+ assertTrue(pFields.contains(3));
+ assertEquals(myP, result.getCustomPartitioner());
+ }
+
+ @Test
+ public void testCustomPartitioningErased() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
+ gprops.setCustomPartitioned(new FieldList(0, 4), myP);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ assertNull(result.getCustomPartitioner());
+ }
+
+ @Test
+ public void testRangePartitioningPreserved1() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;2;5"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(2, StringValue.class, Order.ASCENDING);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setRangePartitioned(o);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertEquals(1, pFields.get(0).intValue());
+ assertEquals(5, pFields.get(1).intValue());
+ assertEquals(2, pFields.get(2).intValue());
+ Ordering pOrder = result.getPartitioningOrdering();
+ assertEquals(3, pOrder.getNumberOfFields());
+ assertEquals(1, pOrder.getFieldNumber(0).intValue());
+ assertEquals(5, pOrder.getFieldNumber(1).intValue());
+ assertEquals(2, pOrder.getFieldNumber(2).intValue());
+ assertEquals(Order.ASCENDING, pOrder.getOrder(0));
+ assertEquals(Order.DESCENDING, pOrder.getOrder(1));
+ assertEquals(Order.ASCENDING, pOrder.getOrder(2));
+ assertEquals(IntValue.class, pOrder.getType(0));
+ assertEquals(LongValue.class, pOrder.getType(1));
+ assertEquals(StringValue.class, pOrder.getType(2));
+ }
+
+ @Test
+ public void testRangePartitioningPreserved2() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1->3; 2->0; 5->1"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(2, StringValue.class, Order.ASCENDING);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setRangePartitioned(o);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
+ FieldList pFields = result.getPartitioningFields();
+ assertEquals(3, pFields.size());
+ assertEquals(3, pFields.get(0).intValue());
+ assertEquals(1, pFields.get(1).intValue());
+ assertEquals(0, pFields.get(2).intValue());
+ Ordering pOrder = result.getPartitioningOrdering();
+ assertEquals(3, pOrder.getNumberOfFields());
+ assertEquals(3, pOrder.getFieldNumber(0).intValue());
+ assertEquals(1, pOrder.getFieldNumber(1).intValue());
+ assertEquals(0, pOrder.getFieldNumber(2).intValue());
+ assertEquals(Order.ASCENDING, pOrder.getOrder(0));
+ assertEquals(Order.DESCENDING, pOrder.getOrder(1));
+ assertEquals(Order.ASCENDING, pOrder.getOrder(2));
+ assertEquals(IntValue.class, pOrder.getType(0));
+ assertEquals(LongValue.class, pOrder.getType(1));
+ assertEquals(StringValue.class, pOrder.getType(2));
+ }
+
+ @Test
+ public void testRangePartitioningErased() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(2, StringValue.class, Order.ASCENDING);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setRangePartitioned(o);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
+ assertNull(result.getPartitioningOrdering());
+ assertNull(result.getPartitioningFields());
+ }
+
+ @Test
+ public void testRebalancingPreserved() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setForcedRebalanced();
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+
+ assertEquals(PartitioningProperty.FORCED_REBALANCED, result.getPartitioning());
+ assertNull(result.getPartitioningFields());
+ }
+
+ @Test
+ public void testUniqueFieldGroupsPreserved1() {
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
+
+ FieldSet set1 = new FieldSet(0, 1, 2);
+ FieldSet set2 = new FieldSet(3, 4);
+ FieldSet set3 = new FieldSet(4, 5, 6, 7);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.addUniqueFieldCombination(set1);
+ gprops.addUniqueFieldCombination(set2);
+ gprops.addUniqueFieldCombination(set3);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+ Set<FieldSet> unique = result.getUniqueFieldCombination();
+ FieldSet expected1 = new FieldSet(0, 1, 2);
+ FieldSet expected2 = new FieldSet(3, 4);
+
+ Assert.assertTrue(unique.size() == 2);
+ Assert.assertTrue(unique.contains(expected1));
+ Assert.assertTrue(unique.contains(expected2));
+ }
+
+ @Test
+ public void testUniqueFieldGroupsPreserved2() {
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->5;1;2;3->6;4"}, null, null, tupleInfo, tupleInfo);
+
+ FieldSet set1 = new FieldSet(0, 1, 2);
+ FieldSet set2 = new FieldSet(3, 4);
+ FieldSet set3 = new FieldSet(4, 5, 6, 7);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.addUniqueFieldCombination(set1);
+ gprops.addUniqueFieldCombination(set2);
+ gprops.addUniqueFieldCombination(set3);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+ Set<FieldSet> unique = result.getUniqueFieldCombination();
+ FieldSet expected1 = new FieldSet(1, 2, 5);
+ FieldSet expected2 = new FieldSet(4, 6);
+
+ Assert.assertTrue(unique.size() == 2);
+ Assert.assertTrue(unique.contains(expected1));
+ Assert.assertTrue(unique.contains(expected2));
+ }
+
+ @Test
+ public void testUniqueFieldGroupsErased() {
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0; 3; 5; 6; 7"}, null, null, tupleInfo, tupleInfo);
+
+ FieldSet set1 = new FieldSet(0, 1, 2);
+ FieldSet set2 = new FieldSet(3, 4);
+ FieldSet set3 = new FieldSet(4, 5, 6, 7);
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.addUniqueFieldCombination(set1);
+ gprops.addUniqueFieldCombination(set2);
+ gprops.addUniqueFieldCombination(set3);
+
+ GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
+ Assert.assertNull(result.getUniqueFieldCombination());
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testInvalidInputIndex() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ GlobalProperties gprops = new GlobalProperties();
+ gprops.setHashPartitioned(new FieldList(0, 1));
+
+ gprops.filterBySemanticProperties(sprops, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
new file mode 100644
index 0000000..52826d6
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+public class GlobalPropertiesMatchingTest {
+
+ @Test
+ public void testMatchingAnyPartitioning() {
+ try {
+
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setAnyPartitioning(new FieldSet(6, 2));
+
+ // match any partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setAnyPartitioning(new FieldList(2, 6));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setAnyPartitioning(new FieldList(6, 2));
+ assertTrue(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setAnyPartitioning(new FieldList(6, 2, 4));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setAnyPartitioning(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp4));
+
+ GlobalProperties gp5 = new GlobalProperties();
+ gp5.setAnyPartitioning(new FieldList(2));
+ assertTrue(req.isMetBy(gp5));
+ }
+
+ // match hash partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setHashPartitioned(new FieldList(2, 6));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setHashPartitioned(new FieldList(6, 2));
+ assertTrue(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setHashPartitioned(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp3));
+ }
+
+ // match range partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
+ assertTrue(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
+ assertTrue(req.isMetBy(gp4));
+ }
+
+ // match custom partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setCustomPartitioned(new FieldList(2, 6), new MockPartitioner());
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setCustomPartitioned(new FieldList(6, 2), new MockPartitioner());
+ assertTrue(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setCustomPartitioned(new FieldList(6, 1), new MockPartitioner());
+ assertFalse(req.isMetBy(gp3));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testMatchingCustomPartitioning() {
+ try {
+ final Partitioner<Tuple2<Long, Integer>> partitioner = new MockPartitioner();
+
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setCustomPartitioned(new FieldSet(6, 2), partitioner);
+
+ // match custom partitionings
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setCustomPartitioned(new FieldList(2, 6), partitioner);
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setCustomPartitioned(new FieldList(6, 2), partitioner);
+ assertTrue(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setCustomPartitioned(new FieldList(6, 2), new MockPartitioner());
+ assertFalse(req.isMetBy(gp3));
+ }
+
+ // cannot match other types of partitionings
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setAnyPartitioning(new FieldList(6, 2));
+ assertFalse(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setHashPartitioned(new FieldList(6, 2));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp3));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testStrictlyMatchingAnyPartitioning() {
+
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setAnyPartitioning(new FieldList(6, 2));
+
+ // match any partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setAnyPartitioning(new FieldList(6, 2));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setAnyPartitioning(new FieldList(2, 6));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setAnyPartitioning(new FieldList(6, 2, 3));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp3.setAnyPartitioning(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp5 = new GlobalProperties();
+ gp4.setAnyPartitioning(new FieldList(2));
+ assertFalse(req.isMetBy(gp4));
+ }
+
+ // match hash partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setHashPartitioned(new FieldList(6, 2));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setHashPartitioned(new FieldList(2, 6));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setHashPartitioned(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp3));
+ }
+
+ // match range partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
+ assertFalse(req.isMetBy(gp4));
+ }
+
+ }
+
+ @Test
+ public void testStrictlyMatchingHashPartitioning() {
+
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setHashPartitioned(new FieldList(6, 2));
+
+ // match any partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setAnyPartitioning(new FieldList(6, 2));
+ assertFalse(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setAnyPartitioning(new FieldList(2, 6));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setAnyPartitioning(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setAnyPartitioning(new FieldList(2));
+ assertFalse(req.isMetBy(gp4));
+ }
+
+ // match hash partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setHashPartitioned(new FieldList(6, 2));
+ assertTrue(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setHashPartitioned(new FieldList(2, 6));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setHashPartitioned(new FieldList(6, 1));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setHashPartitioned(new FieldList(6, 2, 0));
+ assertFalse(req.isMetBy(gp4));
+ }
+
+ // match range partitioning
+ {
+ GlobalProperties gp1 = new GlobalProperties();
+ gp1.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp1));
+
+ GlobalProperties gp2 = new GlobalProperties();
+ gp2.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp2));
+
+ GlobalProperties gp3 = new GlobalProperties();
+ gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
+ assertFalse(req.isMetBy(gp3));
+
+ GlobalProperties gp4 = new GlobalProperties();
+ gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
+ assertFalse(req.isMetBy(gp4));
+ }
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
new file mode 100644
index 0000000..0868720
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.junit.Test;
+
+public class GlobalPropertiesPushdownTest {
+
+ @Test
+ public void testAnyPartitioningPushedDown() {
+ try {
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setAnyPartitioning(new FieldSet(3, 1));
+
+ RequestedGlobalProperties preserved = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
+ assertEquals(PartitioningProperty.ANY_PARTITIONING, preserved.getPartitioning());
+ assertTrue(preserved.getPartitionedFields().isValidSubset(new FieldSet(1, 3)));
+
+ RequestedGlobalProperties nonPreserved = req.filterBySemanticProperties(getNonePreservingSemProps(), 0);
+ assertTrue(nonPreserved == null || nonPreserved.isTrivial());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testHashPartitioningPushedDown() {
+ try {
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setHashPartitioned(new FieldSet(3, 1));
+
+ RequestedGlobalProperties preserved = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, preserved.getPartitioning());
+ assertTrue(preserved.getPartitionedFields().isValidSubset(new FieldSet(1, 3)));
+
+ RequestedGlobalProperties nonPreserved = req.filterBySemanticProperties(getNonePreservingSemProps(), 0);
+ assertTrue(nonPreserved == null || nonPreserved.isTrivial());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningNotPushedDown() {
+ try {
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setCustomPartitioned(new FieldSet(3, 1), new MockPartitioner());
+
+ RequestedGlobalProperties pushedDown = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
+ assertTrue(pushedDown == null || pushedDown.isTrivial());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testForcedReblancingNotPushedDown() {
+ try {
+ RequestedGlobalProperties req = new RequestedGlobalProperties();
+ req.setForceRebalancing();
+
+ RequestedGlobalProperties pushedDown = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
+ assertTrue(pushedDown == null || pushedDown.isTrivial());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static SemanticProperties getAllPreservingSemProps() {
+ return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
+ }
+
+ private static SemanticProperties getNonePreservingSemProps() {
+ return new SingleInputSemanticProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
new file mode 100644
index 0000000..1ff62ed
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.StringValue;
+import org.junit.Test;
+
+public class LocalPropertiesFilteringTest {
+
+ private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
+ new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ @Test
+ public void testAllErased1() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+ lProps = lProps.addUniqueFields(new FieldSet(5,6));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testAllErased2() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"5"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testGroupingPreserved1() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNotNull(filtered.getGroupedFields());
+ assertEquals(3, filtered.getGroupedFields().size());
+ assertTrue(filtered.getGroupedFields().contains(0));
+ assertTrue(filtered.getGroupedFields().contains(2));
+ assertTrue(filtered.getGroupedFields().contains(3));
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testGroupingPreserved2() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0;3->7"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNotNull(filtered.getGroupedFields());
+ assertEquals(3, filtered.getGroupedFields().size());
+ assertTrue(filtered.getGroupedFields().contains(4));
+ assertTrue(filtered.getGroupedFields().contains(0));
+ assertTrue(filtered.getGroupedFields().contains(7));
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testGroupingErased() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testSortingPreserved1() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(2, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(0, StringValue.class, Order.DESCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ LocalProperties lProps = LocalProperties.forOrdering(o);
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldList gFields = filtered.getGroupedFields();
+ Ordering order = filtered.getOrdering();
+
+ assertNotNull(gFields);
+ assertEquals(3, gFields.size());
+ assertTrue(gFields.contains(0));
+ assertTrue(gFields.contains(2));
+ assertTrue(gFields.contains(5));
+ assertNotNull(order);
+ assertEquals(3, order.getNumberOfFields());
+ assertEquals(2, order.getFieldNumber(0).intValue());
+ assertEquals(0, order.getFieldNumber(1).intValue());
+ assertEquals(5, order.getFieldNumber(2).intValue());
+ assertEquals(Order.ASCENDING, order.getOrder(0));
+ assertEquals(Order.DESCENDING, order.getOrder(1));
+ assertEquals(Order.DESCENDING, order.getOrder(2));
+ assertEquals(IntValue.class, order.getType(0));
+ assertEquals(StringValue.class, order.getType(1));
+ assertEquals(LongValue.class, order.getType(2));
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testSortingPreserved2() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(2, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(0, StringValue.class, Order.DESCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ LocalProperties lProps = LocalProperties.forOrdering(o);
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldList gFields = filtered.getGroupedFields();
+ Ordering order = filtered.getOrdering();
+
+ assertNotNull(gFields);
+ assertEquals(3, gFields.size());
+ assertTrue(gFields.contains(3));
+ assertTrue(gFields.contains(7));
+ assertTrue(gFields.contains(1));
+ assertNotNull(order);
+ assertEquals(3, order.getNumberOfFields());
+ assertEquals(7, order.getFieldNumber(0).intValue());
+ assertEquals(3, order.getFieldNumber(1).intValue());
+ assertEquals(1, order.getFieldNumber(2).intValue());
+ assertEquals(Order.ASCENDING, order.getOrder(0));
+ assertEquals(Order.DESCENDING, order.getOrder(1));
+ assertEquals(Order.DESCENDING, order.getOrder(2));
+ assertEquals(IntValue.class, order.getType(0));
+ assertEquals(StringValue.class, order.getType(1));
+ assertEquals(LongValue.class, order.getType(2));
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testSortingPreserved3() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(2, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(0, StringValue.class, Order.DESCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ LocalProperties lProps = LocalProperties.forOrdering(o);
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldList gFields = filtered.getGroupedFields();
+ Ordering order = filtered.getOrdering();
+
+ assertNotNull(gFields);
+ assertEquals(2, gFields.size());
+ assertTrue(gFields.contains(0));
+ assertTrue(gFields.contains(2));
+ assertNotNull(order);
+ assertEquals(2, order.getNumberOfFields());
+ assertEquals(2, order.getFieldNumber(0).intValue());
+ assertEquals(0, order.getFieldNumber(1).intValue());
+ assertEquals(Order.ASCENDING, order.getOrder(0));
+ assertEquals(Order.DESCENDING, order.getOrder(1));
+ assertEquals(IntValue.class, order.getType(0));
+ assertEquals(StringValue.class, order.getType(1));
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testSortingPreserved4() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(2, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(0, StringValue.class, Order.DESCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ LocalProperties lProps = LocalProperties.forOrdering(o);
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldList gFields = filtered.getGroupedFields();
+ Ordering order = filtered.getOrdering();
+
+ assertNotNull(gFields);
+ assertEquals(1, gFields.size());
+ assertTrue(gFields.contains(7));
+ assertNotNull(order);
+ assertEquals(1, order.getNumberOfFields());
+ assertEquals(7, order.getFieldNumber(0).intValue());
+ assertEquals(Order.ASCENDING, order.getOrder(0));
+ assertEquals(IntValue.class, order.getType(0));
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testSortingErased() {
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(2, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(0, StringValue.class, Order.DESCENDING);
+ o.appendOrdering(5, LongValue.class, Order.DESCENDING);
+ LocalProperties lProps = LocalProperties.forOrdering(o);
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldList gFields = filtered.getGroupedFields();
+ Ordering order = filtered.getOrdering();
+
+ assertNull(gFields);
+ assertNull(order);
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test
+ public void testUniqueFieldsPreserved1() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = new LocalProperties();
+ lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+ lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldSet expected1 = new FieldSet(0,1,2);
+ FieldSet expected2 = new FieldSet(3,4);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNotNull(filtered.getUniqueFields());
+ assertEquals(2, filtered.getUniqueFields().size());
+ assertTrue(filtered.getUniqueFields().contains(expected1));
+ assertTrue(filtered.getUniqueFields().contains(expected2));
+ }
+
+ @Test
+ public void testUniqueFieldsPreserved2() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = LocalProperties.forGrouping(new FieldList(1,2));
+ lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+ lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldSet expected1 = new FieldSet(0,1,2);
+ FieldSet expected2 = new FieldSet(3,4);
+
+ assertNull(filtered.getOrdering());
+ assertNotNull(filtered.getGroupedFields());
+ assertEquals(2, filtered.getGroupedFields().size());
+ assertTrue(filtered.getGroupedFields().contains(1));
+ assertTrue(filtered.getGroupedFields().contains(2));
+ assertNotNull(filtered.getUniqueFields());
+ assertEquals(2, filtered.getUniqueFields().size());
+ assertTrue(filtered.getUniqueFields().contains(expected1));
+ assertTrue(filtered.getUniqueFields().contains(expected2));
+ }
+
+ @Test
+ public void testUniqueFieldsPreserved3() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->7;1->6;2->5;3->4;4->3"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = new LocalProperties();
+ lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+ lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+ FieldSet expected1 = new FieldSet(5,6,7);
+ FieldSet expected2 = new FieldSet(3,4);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNotNull(filtered.getUniqueFields());
+ assertEquals(2, filtered.getUniqueFields().size());
+ assertTrue(filtered.getUniqueFields().contains(expected1));
+ assertTrue(filtered.getUniqueFields().contains(expected2));
+ }
+
+ @Test
+ public void testUniqueFieldsErased() {
+
+ SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lProps = new LocalProperties();
+ lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
+ lProps = lProps.addUniqueFields(new FieldSet(3,4));
+ lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
+
+ LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
+
+ assertNull(filtered.getGroupedFields());
+ assertNull(filtered.getOrdering());
+ assertNull(filtered.getUniqueFields());
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testInvalidInputIndex() {
+
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ LocalProperties lprops = LocalProperties.forGrouping(new FieldList(0,1));
+
+ lprops.filterBySemanticProperties(sprops, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
new file mode 100644
index 0000000..74126f8
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.types.Key;
+
+import java.io.IOException;
+
+@SuppressWarnings("serial")
+public class MockDistribution implements DataDistribution {
+
+ @Override
+ public Key<?>[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+ return new Key<?>[0];
+ }
+
+ @Override
+ public int getNumberOfFields() {
+ return 0;
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
new file mode 100644
index 0000000..2b2ab14
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.tuple.Tuple2;
+
+class MockPartitioner implements Partitioner<Tuple2<Long, Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public int partition(Tuple2<Long, Integer> key, int numPartitions) {
+ return 0;
+ }
+}
\ No newline at end of file
[21/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
deleted file mode 100644
index 8720aa7..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichJoinFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-/**
-* Tests that validate optimizer choices when using operators that are requesting certain specific execution
-* strategies.
-*/
-@SuppressWarnings("serial")
-public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
-
- private static final String JOIN_WITH_INVARIANT_NAME = "Test Join Invariant";
- private static final String JOIN_WITH_SOLUTION_SET = "Test Join SolutionSet";
- private static final String NEXT_WORKSET_REDUCER_NAME = "Test Reduce Workset";
- private static final String SOLUTION_DELTA_MAPPER_NAME = "Test Map Delta";
-
- @Test
- public void testJavaApiWithDeferredSoltionSetUpdateWithMapper() {
- try {
- Plan plan = getJavaTestPlan(false, true);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
- SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
-
-
- // verify reducer
- assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
- assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
-
- // currently, the system may partition before or after the mapper
- ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
- ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
-
- assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
- (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- @Test
- public void testJavaApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
- try {
- Plan plan = getJavaTestPlan(false, false);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
-
- // verify reducer
- assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
- assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
-
- // verify solution delta
- assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- @Test
- public void testJavaApiWithDirectSoltionSetUpdate() {
- try {
- Plan plan = getJavaTestPlan(true, false);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput1());
- assertEquals(new FieldList(1, 2), joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
-
- // verify reducer
- assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
- assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
-
-
- // verify solution delta
- assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
-
- @Test
- public void testRejectPlanIfSolutionSetKeysAndJoinKeysDontMatch() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> solutionSetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Solution Set");
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> worksetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Workset");
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> invariantInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Invariant Input");
-
- DeltaIteration<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> iter = solutionSetInput.iterateDelta(worksetInput, 100, 1, 2);
-
-
- DataSet<Tuple3<Long, Long, Long>> result =
-
- iter.getWorkset().join(invariantInput)
- .where(1, 2)
- .equalTo(1, 2)
- .with(new JoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
- public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
- return first;
- }
- });
-
- try {
- result.join(iter.getSolutionSet())
- .where(1, 0)
- .equalTo(0, 2)
- .with(new JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
- public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
- return second;
- }
- });
- fail("The join should be rejected with key type mismatches.");
- }
- catch (InvalidProgramException e) {
- // expected!
- }
-
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- private Plan getJavaTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> solutionSetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Solution Set");
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> worksetInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Workset");
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> invariantInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Invariant Input");
-
- DeltaIteration<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> iter = solutionSetInput.iterateDelta(worksetInput, 100, 1, 2);
-
-
- DataSet<Tuple3<Long, Long, Long>> joinedWithSolutionSet =
-
- iter.getWorkset().join(invariantInput)
- .where(1, 2)
- .equalTo(1, 2)
- .with(new RichJoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
- public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
- return first;
- }
- })
- .name(JOIN_WITH_INVARIANT_NAME)
-
- .join(iter.getSolutionSet())
- .where(1, 0)
- .equalTo(1, 2)
- .with(new RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
- public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
- return second;
- }
- })
- .name(JOIN_WITH_SOLUTION_SET)
- .withForwardedFieldsSecond(joinPreservesSolutionSet ? new String[] {"0->0", "1->1", "2->2" } : null);
-
- DataSet<Tuple3<Long, Long, Long>> nextWorkset = joinedWithSolutionSet.groupBy(1, 2)
- .reduceGroup(new RichGroupReduceFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>() {
- public void reduce(Iterable<Tuple3<Long, Long, Long>> values, Collector<Tuple3<Long, Long, Long>> out) {}
- })
- .name(NEXT_WORKSET_REDUCER_NAME)
- .withForwardedFields("1->1","2->2","0->0");
-
-
- DataSet<Tuple3<Long, Long, Long>> nextSolutionSet = mapBeforeSolutionDelta ?
- joinedWithSolutionSet.map(new RichMapFunction<Tuple3<Long, Long, Long>,Tuple3<Long, Long, Long>>() { public Tuple3<Long, Long, Long> map(Tuple3<Long, Long, Long> value) { return value; } })
- .name(SOLUTION_DELTA_MAPPER_NAME).withForwardedFields("0->0","1->1","2->2") :
- joinedWithSolutionSet;
-
- iter.closeWith(nextSolutionSet, nextWorkset)
- .print();
-
- return env.createProgramPlan();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
deleted file mode 100644
index 23f8897..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class CoGroupGlobalPropertiesCompatibilityTest {
-
- @Test
- public void checkCompatiblePartitionings() {
- try {
- final FieldList keysLeft = new FieldList(1, 4);
- final FieldList keysRight = new FieldList(3, 1);
-
- CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
-
- // test compatible hash partitioning
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setHashPartitioned(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setHashPartitioned(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setHashPartitioned(keysLeft);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setHashPartitioned(keysRight);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test compatible custom partitioning
- {
- Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setCustomPartitioned(keysLeft, part);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setCustomPartitioned(keysRight, part);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test custom partitioning matching any partitioning
- {
- Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void checkInompatiblePartitionings() {
- try {
- final FieldList keysLeft = new FieldList(1);
- final FieldList keysRight = new FieldList(3);
-
- final Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
- final Partitioner<Object> part2 = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
-
- // test incompatible hash with custom partitioning
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setHashPartitioned(keysLeft);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test incompatible custom partitionings
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part2);
-
- assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
deleted file mode 100644
index e7807c9..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.configuration.Configuration;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class CoGroupOnConflictingPartitioningsTest extends CompilerTestBase {
-
- @Test
- public void testRejectCoGroupOnHashAndRangePartitioning() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
-
- Configuration cfg = new Configuration();
- cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
- cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
-
- input.coGroup(input).where(0).equalTo(0)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
- .withParameters(cfg)
- .print();
-
- Plan p = env.createProgramPlan();
- try {
- compileNoStats(p);
- fail("This should fail with an exception");
- }
- catch (CompilerException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
deleted file mode 100644
index 839f0a1..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class JoinGlobalPropertiesCompatibilityTest {
-
- @Test
- public void checkCompatiblePartitionings() {
- try {
- final FieldList keysLeft = new FieldList(1, 4);
- final FieldList keysRight = new FieldList(3, 1);
-
- SortMergeJoinDescriptor descr = new SortMergeJoinDescriptor(keysLeft, keysRight);
-
- // test compatible hash partitioning
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setHashPartitioned(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setHashPartitioned(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setHashPartitioned(keysLeft);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setHashPartitioned(keysRight);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test compatible custom partitioning
- {
- Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setCustomPartitioned(keysLeft, part);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setCustomPartitioned(keysRight, part);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test custom partitioning matching any partitioning
- {
- Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void checkInompatiblePartitionings() {
- try {
- final FieldList keysLeft = new FieldList(1);
- final FieldList keysRight = new FieldList(3);
-
- final Partitioner<Object> part = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
- final Partitioner<Object> part2 = new Partitioner<Object>() {
- @Override
- public int partition(Object key, int numPartitions) {
- return 0;
- }
- };
-
- SortMergeJoinDescriptor descr = new SortMergeJoinDescriptor(keysLeft, keysRight);
-
- // test incompatible hash with custom partitioning
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setHashPartitioned(keysLeft);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part);
-
- assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
-
- // test incompatible custom partitionings
- {
- RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
- reqLeft.setAnyPartitioning(keysLeft);
- RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
- reqRight.setAnyPartitioning(keysRight);
-
- GlobalProperties propsLeft = new GlobalProperties();
- propsLeft.setCustomPartitioned(keysLeft, part);
- GlobalProperties propsRight = new GlobalProperties();
- propsRight.setCustomPartitioned(keysRight, part2);
-
- assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
deleted file mode 100644
index 9171cc7..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.configuration.Configuration;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class JoinOnConflictingPartitioningsTest extends CompilerTestBase {
-
- @Test
- public void testRejectJoinOnHashAndRangePartitioning() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
-
- Configuration cfg = new Configuration();
- cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
- cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
-
- input.join(input).where(0).equalTo(0)
- .withParameters(cfg)
- .print();
-
- Plan p = env.createProgramPlan();
- try {
- compileNoStats(p);
- fail("This should fail with an exception");
- }
- catch (CompilerException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
deleted file mode 100644
index 2c1574b..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/plan/ChannelTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.OperatorInformation;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.core.fs.Path;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.flink.api.java.io.TextInputFormat;
-
-public class ChannelTest {
-
- @Test
- public void testGetEstimatesNoReplicationFactor() {
- final long NUM_RECORD = 1001;
- final long SIZE = 467131;
-
- DataSourceNode source = getSourceNode();
- SourcePlanNode planNode = new SourcePlanNode(source, "test node");
- Channel channel = new Channel(planNode);
-
- // no estimates here
- Assert.assertEquals(-1, channel.getEstimatedOutputSize());
- Assert.assertEquals(-1, channel.getEstimatedNumRecords());
-
- // set estimates
- source.setEstimatedNumRecords(NUM_RECORD);
- source.setEstimatedOutputSize(SIZE);
- Assert.assertEquals(SIZE, channel.getEstimatedOutputSize());
- Assert.assertEquals(NUM_RECORD, channel.getEstimatedNumRecords());
- }
-
- @Test
- public void testGetEstimatesWithReplicationFactor() {
- final long NUM_RECORD = 1001;
- final long SIZE = 467131;
-
- final int REPLICATION = 23;
-
- DataSourceNode source = getSourceNode();
- SourcePlanNode planNode = new SourcePlanNode(source, "test node");
- Channel channel = new Channel(planNode);
- channel.setReplicationFactor(REPLICATION);
-
- // no estimates here
- Assert.assertEquals(-1, channel.getEstimatedOutputSize());
- Assert.assertEquals(-1, channel.getEstimatedNumRecords());
-
- // set estimates
- source.setEstimatedNumRecords(NUM_RECORD);
- source.setEstimatedOutputSize(SIZE);
- Assert.assertEquals(SIZE * REPLICATION, channel.getEstimatedOutputSize());
- Assert.assertEquals(NUM_RECORD * REPLICATION, channel.getEstimatedNumRecords());
- }
-
-
-// private static final OptimizerNode getSingleInputNode() {
-// return new MapNode(new MapOperatorBase<String, String, GenericMap<String,String>>(
-// new IdentityMapper<String>(),
-// new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
-// "map"));
-// }
-
- private static final DataSourceNode getSourceNode() {
- return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>(
- new TextInputFormat(new Path("/ignored")),
- new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO),
- "source"));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
deleted file mode 100644
index 366d10d..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/plandump/NumberFormattingTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plandump;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Test;
-
-public class NumberFormattingTest {
-
- @Test
- public void testFormatNumberNoDigit() {
- assertEquals("0.0", PlanJSONDumpGenerator.formatNumber(0));
- assertEquals("0.00", PlanJSONDumpGenerator.formatNumber(0.0000000001));
- assertEquals("-1.0", PlanJSONDumpGenerator.formatNumber(-1.0));
- assertEquals("1.00", PlanJSONDumpGenerator.formatNumber(1));
- assertEquals("17.00", PlanJSONDumpGenerator.formatNumber(17));
- assertEquals("17.44", PlanJSONDumpGenerator.formatNumber(17.44));
- assertEquals("143.00", PlanJSONDumpGenerator.formatNumber(143));
- assertEquals("143.40", PlanJSONDumpGenerator.formatNumber(143.4));
- assertEquals("143.50", PlanJSONDumpGenerator.formatNumber(143.5));
- assertEquals("143.60", PlanJSONDumpGenerator.formatNumber(143.6));
- assertEquals("143.45", PlanJSONDumpGenerator.formatNumber(143.45));
- assertEquals("143.55", PlanJSONDumpGenerator.formatNumber(143.55));
- assertEquals("143.65", PlanJSONDumpGenerator.formatNumber(143.65));
- assertEquals("143.66", PlanJSONDumpGenerator.formatNumber(143.655));
-
- assertEquals("1.13 K", PlanJSONDumpGenerator.formatNumber(1126.0));
- assertEquals("11.13 K", PlanJSONDumpGenerator.formatNumber(11126.0));
- assertEquals("118.13 K", PlanJSONDumpGenerator.formatNumber(118126.0));
-
- assertEquals("1.44 M", PlanJSONDumpGenerator.formatNumber(1435126.0));
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
deleted file mode 100644
index 7fea8a6..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyCoGroupFunction.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichCoGroupFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-
-public class DummyCoGroupFunction<L, R> extends RichCoGroupFunction<L, R, Tuple2<L, R>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void coGroup(Iterable<L> first, Iterable<R> second, Collector<Tuple2<L, R>> out) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
deleted file mode 100644
index 6be8a24..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyFlatJoinFunction.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichFlatJoinFunction;
-import org.apache.flink.util.Collector;
-
-public class DummyFlatJoinFunction<T> extends RichFlatJoinFunction<T, T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void join(T first, T second, Collector<T> out) {
- out.collect(null);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
deleted file mode 100644
index 44d3695..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/DummyReducer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichReduceFunction;
-
-public class DummyReducer<T> extends RichReduceFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T reduce(T a, T b) {
- return a;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
deleted file mode 100644
index 0316463..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityFlatMapper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.util.Collector;
-
-public class IdentityFlatMapper<T> implements FlatMapFunction<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(T value, Collector<T> out) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
deleted file mode 100644
index 11fd044..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityGroupReducer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable;
-import org.apache.flink.util.Collector;
-
-
-@Combinable
-public class IdentityGroupReducer<T> extends RichGroupReduceFunction<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<T> values, Collector<T> out) {
- for (T next : values) {
- out.collect(next);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
deleted file mode 100644
index f335846..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityKeyExtractor.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.java.functions.KeySelector;
-
-public class IdentityKeyExtractor<T> implements KeySelector<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T getKey(T value) {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
deleted file mode 100644
index 025b4d8..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityMapper.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-
-public class IdentityMapper<T> extends RichMapFunction<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T map(T value) {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
deleted file mode 100644
index 6efbef1..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/IdentityPartitionerMapper.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichMapPartitionFunction;
-import org.apache.flink.util.Collector;
-
-public class IdentityPartitionerMapper<T> extends RichMapPartitionFunction<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void mapPartition(Iterable<T> values, Collector<T> out) {
- for (T in : values) {
- out.collect(in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
deleted file mode 100644
index 39c0e1b..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/SelectOneReducer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichReduceFunction;
-
-public class SelectOneReducer<T> extends RichReduceFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T reduce(T value1, T value2) throws Exception {
- return value1;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
deleted file mode 100644
index 48d13ca..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/testfunctions/Top1GroupReducer.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.testfunctions;
-
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable;
-import org.apache.flink.util.Collector;
-
-
-@Combinable
-public class Top1GroupReducer<T> extends RichGroupReduceFunction<T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<T> values, Collector<T> out) {
- out.collect(values.iterator().next());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
deleted file mode 100644
index 6a84c44..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCoGroupStub.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-import org.apache.flink.api.java.record.functions.CoGroupFunction;
-import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("deprecation")
-public class DummyCoGroupStub extends CoGroupFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) {
- while (records1.hasNext()) {
- out.collect(records1.next());
- }
-
- while (records2.hasNext()) {
- out.collect(records2.next());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
deleted file mode 100644
index 8ee2285..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyCrossStub.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import org.apache.flink.api.java.record.functions.CrossFunction;
-import org.apache.flink.types.Record;
-
-@SuppressWarnings("deprecation")
-public class DummyCrossStub extends CrossFunction {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Record cross(Record first, Record second) throws Exception {
- return first;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
deleted file mode 100644
index 0c816e7..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyInputFormat.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.java.record.io.DelimitedInputFormat;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.Record;
-
-public final class DummyInputFormat extends DelimitedInputFormat {
- private static final long serialVersionUID = 1L;
-
- private final IntValue integer = new IntValue(1);
-
- @Override
- public Record readRecord(Record target, byte[] bytes, int offset, int numBytes) {
- target.setField(0, this.integer);
- target.setField(1, this.integer);
- return target;
- }
-
- @Override
- public FileBaseStatistics getStatistics(BaseStatistics cachedStatistics) {
- return (cachedStatistics instanceof FileBaseStatistics) ? (FileBaseStatistics) cachedStatistics : null;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
deleted file mode 100644
index d00be6e..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyMatchStub.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.java.record.functions.JoinFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsFirstExcept;
-import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("deprecation")
-@ConstantFieldsFirstExcept({})
-public class DummyMatchStub extends JoinFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void join(Record value1, Record value2, Collector<Record> out) throws Exception {
- out.collect(value1);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
deleted file mode 100644
index 444b48e..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyNonPreservingMatchStub.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.java.record.functions.JoinFunction;
-import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("deprecation")
-public class DummyNonPreservingMatchStub extends JoinFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void join(Record value1, Record value2, Collector<Record> out) throws Exception {
- out.collect(value1);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
deleted file mode 100644
index 1bbe24c..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/DummyOutputFormat.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.util;
-
-
-import org.apache.flink.api.java.record.io.DelimitedOutputFormat;
-import org.apache.flink.types.Record;
-
-
-public final class DummyOutputFormat extends DelimitedOutputFormat {
- private static final long serialVersionUID = 1L;
-
- @Override
- public int serializeRecord(Record rec, byte[] target) throws Exception {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
deleted file mode 100644
index cccc6cb..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityMap.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.java.record.functions.MapFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsExcept;
-import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("deprecation")
-@ConstantFieldsExcept({})
-public final class IdentityMap extends MapFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void map(Record record, Collector<Record> out) throws Exception {
- out.collect(record);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
deleted file mode 100644
index f45745d..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/util/IdentityReduce.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-import org.apache.flink.api.java.record.functions.ReduceFunction;
-import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsExcept;
-import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-
-@SuppressWarnings("deprecation")
-@ConstantFieldsExcept({})
-public final class IdentityReduce extends ReduceFunction implements Serializable {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
- while (records.hasNext()) {
- out.collect(records.next());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/resources/log4j-test.properties b/flink-compiler/src/test/resources/log4j-test.properties
deleted file mode 100644
index 2fb9345..0000000
--- a/flink-compiler/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,19 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=OFF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/resources/log4j.properties b/flink-compiler/src/test/resources/log4j.properties
deleted file mode 100644
index fa3f937..0000000
--- a/flink-compiler/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# This file ensures that tests executed from the IDE show log output
-
-log4j.rootLogger=INFO, console
-
-# Log all infos in the given file
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target = System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/resources/logback-test.xml b/flink-compiler/src/test/resources/logback-test.xml
deleted file mode 100644
index 8b3bb27..0000000
--- a/flink-compiler/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 7ee7f25..b826c45 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -61,7 +61,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
+ <artifactId>flink-optimizer</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/pom.xml
----------------------------------------------------------------------
diff --git a/flink-optimizer/pom.xml b/flink-optimizer/pom.xml
new file mode 100644
index 0000000..55764e9
--- /dev/null
+++ b/flink-optimizer/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-optimizer</artifactId>
+ <name>flink-optimizer</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerException.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerException.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerException.java
new file mode 100644
index 0000000..2f99ddb
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerException.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+/**
+ * An exception that is thrown by the Optimizer when encountering an illegal condition.
+ */
+public class CompilerException extends RuntimeException {
+
+ private static final long serialVersionUID = 3810067304570563755L;
+
+ /**
+ * Creates a compiler exception with no message and no cause.
+ */
+ public CompilerException() {}
+
+ /**
+ * Creates a compiler exception with the given message and no cause.
+ *
+ * @param message
+ * The message for the exception.
+ */
+ public CompilerException(String message) {
+ super(message);
+ }
+
+ /**
+ * Creates a compiler exception with the given cause and no message.
+ *
+ * @param cause
+ * The <tt>Throwable</tt> that caused this exception.
+ */
+ public CompilerException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Creates a compiler exception with the given message and cause.
+ *
+ * @param message
+ * The message for the exception.
+ * @param cause
+ * The <tt>Throwable</tt> that caused this exception.
+ */
+ public CompilerException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
new file mode 100644
index 0000000..78e47a0
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+/**
+ * An exception that is thrown by the Optimizer when encountering
+ * a problem during the optimizer post pass. This is a dedicated exception
+ * because it is thrown by user-specified optimizer extensions.
+ */
+public class CompilerPostPassException extends CompilerException {
+
+ private static final long serialVersionUID = -322650826288034623L;
+
+ /**
+ * Creates a post pass exception with no message and no cause.
+ */
+ public CompilerPostPassException() {}
+
+ /**
+ * Creates a post pass exception with the given message and no cause.
+ *
+ * @param message The message for the exception.
+ */
+ public CompilerPostPassException(String message) {
+ super(message);
+ }
+
+ /**
+ * Creates a post pass exception with the given cause and no message.
+ *
+ * @param cause The <tt>Throwable</tt> that caused this exception.
+ */
+ public CompilerPostPassException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Creates a post pass exception with the given message and cause.
+ *
+ * @param message The message for the exception.
+ * @param cause The <tt>Throwable</tt> that caused this exception.
+ */
+ public CompilerPostPassException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
[34/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
deleted file mode 100644
index b3c083a..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-public final class AllGroupWithPartialPreGroupProperties extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.ALL_GROUP_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
- // locally connected, directly instantiate
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
- in, DriverStrategy.ALL_GROUP_REDUCE);
- } else {
- // non forward case.plug in a combiner
- Channel toCombiner = new Channel(in.getSource());
- toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- // create an input node for combine with same DOP as input node
- GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
-
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
- combiner.setCosts(new Costs(0, 0));
- combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
-
- Channel toReducer = new Channel(combiner);
- toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
- in.getShipStrategySortOrder(), in.getDataExchangeMode());
-
- toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
- toReducer, DriverStrategy.ALL_GROUP_REDUCE);
- }
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
deleted file mode 100644
index a172a60..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.ReduceNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-public final class AllReduceProperties extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.ALL_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
- // locally connected, directly instantiate
- return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
- in, DriverStrategy.ALL_REDUCE);
- } else {
- // non forward case.plug in a combiner
- Channel toCombiner = new Channel(in.getSource());
- toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- // create an input node for combine with same DOP as input node
- ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
-
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
- combiner.setCosts(new Costs(0, 0));
- combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
-
- Channel toReducer = new Channel(combiner);
- toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
- in.getShipStrategySortOrder(), in.getDataExchangeMode());
- toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
- in.getLocalStrategySortOrder());
-
- return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
- toReducer, DriverStrategy.ALL_REDUCE);
- }
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return new GlobalProperties();
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return new LocalProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
deleted file mode 100644
index 8cc517e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class BinaryUnionOpDescriptor extends OperatorDescriptorDual {
-
- public BinaryUnionOpDescriptor() {
- super();
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.UNION;
- }
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
- return Collections.emptyList();
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- return Collections.emptyList();
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- return new BinaryUnionPlanNode((BinaryUnionNode) node, in1, in2);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- GlobalProperties newProps = new GlobalProperties();
-
- if (in1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- in2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- in1.getPartitioningFields().equals(in2.getPartitioningFields()))
- {
- newProps.setHashPartitioned(in1.getPartitioningFields());
- }
-
- return newProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- // all local properties are destroyed
- return new LocalProperties();
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2) {
- return true;
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2) {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
deleted file mode 100644
index f48e297..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-
-
-public abstract class CartesianProductDescriptor extends OperatorDescriptorDual {
-
- private final boolean allowBroadcastFirst;
- private final boolean allowBroadcastSecond;
-
-
- protected CartesianProductDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
- if (!(allowBroadcastFirst | allowBroadcastSecond)) {
- throw new IllegalArgumentException();
- }
-
- this.allowBroadcastFirst = allowBroadcastFirst;
- this.allowBroadcastSecond = allowBroadcastSecond;
- }
-
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
- ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
-
- if (this.allowBroadcastFirst) {
- // replicate first
- RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
- replicated1.setFullyReplicated();
- RequestedGlobalProperties any2 = new RequestedGlobalProperties();
- pairs.add(new GlobalPropertiesPair(replicated1, any2));
- }
-
- if (this.allowBroadcastSecond) {
- // replicate second
- RequestedGlobalProperties any1 = new RequestedGlobalProperties();
- RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
- replicated2.setFullyReplicated();
- pairs.add(new GlobalPropertiesPair(any1, replicated2));
- }
-
- return pairs;
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- // all properties are possible
- return Collections.singletonList(new LocalPropertiesPair(
- new RequestedLocalProperties(), new RequestedLocalProperties()));
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2) {
- return true;
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2) {
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- return new DualInputPlanNode(node, "Cross("+node.getOperator().getName()+")", in1, in2, getStrategy());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- GlobalProperties gp = GlobalProperties.combine(in1, in2);
- if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
- gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gp.clearUniqueFieldCombinations();
- return gp;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
deleted file mode 100644
index 368944e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-public class CoGroupDescriptor extends OperatorDescriptorDual {
-
- private final Ordering ordering1; // ordering on the first input
- private final Ordering ordering2; // ordering on the second input
-
- private Partitioner<?> customPartitioner;
-
-
- public CoGroupDescriptor(FieldList keys1, FieldList keys2) {
- this(keys1, keys2, null, null);
- }
-
- public CoGroupDescriptor(FieldList keys1, FieldList keys2, Ordering additionalOrdering1, Ordering additionalOrdering2) {
- super(keys1, keys2);
-
- // if we have an additional ordering, construct the ordering to have primarily the grouping fields
- if (additionalOrdering1 != null) {
- this.ordering1 = new Ordering();
- for (Integer key : this.keys1) {
- this.ordering1.appendOrdering(key, null, Order.ANY);
- }
-
- // and next the additional order fields
- for (int i = 0; i < additionalOrdering1.getNumberOfFields(); i++) {
- Integer field = additionalOrdering1.getFieldNumber(i);
- Order order = additionalOrdering1.getOrder(i);
- this.ordering1.appendOrdering(field, additionalOrdering1.getType(i), order);
- }
- } else {
- this.ordering1 = Utils.createOrdering(this.keys1);
- }
-
- // if we have an additional ordering, construct the ordering to have primarily the grouping fields
- if (additionalOrdering2 != null) {
- this.ordering2 = new Ordering();
- for (Integer key : this.keys2) {
- this.ordering2.appendOrdering(key, null, Order.ANY);
- }
-
- // and next the additional order fields
- for (int i = 0; i < additionalOrdering2.getNumberOfFields(); i++) {
- Integer field = additionalOrdering2.getFieldNumber(i);
- Order order = additionalOrdering2.getOrder(i);
- this.ordering2.appendOrdering(field, additionalOrdering2.getType(i), order);
- }
- } else {
- this.ordering2 = Utils.createOrdering(this.keys2);
- }
- }
-
- public void setCustomPartitioner(Partitioner<?> customPartitioner) {
- this.customPartitioner = customPartitioner;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.CO_GROUP;
- }
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
-
- if (this.customPartitioner == null) {
-
- // we accept compatible partitionings of any type
- RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
- RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
- partitioned_left_any.setAnyPartitioning(this.keys1);
- partitioned_right_any.setAnyPartitioning(this.keys2);
-
- // add strict hash partitioning of both inputs on their full key sets
- RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
- RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
- partitioned_left_hash.setHashPartitioned(this.keys1);
- partitioned_right_hash.setHashPartitioned(this.keys2);
-
- return Arrays.asList(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any),
- new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
- }
- else {
- RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
- partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);
-
- RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
- partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);
-
- return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
- }
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- RequestedLocalProperties sort1 = new RequestedLocalProperties(this.ordering1);
- RequestedLocalProperties sort2 = new RequestedLocalProperties(this.ordering2);
- return Collections.singletonList(new LocalPropertiesPair(sort1, sort2));
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2)
- {
-
- if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
-
- // both are hash partitioned, check that partitioning fields are equivalently chosen
- return checkEquivalentFieldPositionsInKeyFields(
- produced1.getPartitioningFields(), produced2.getPartitioningFields());
-
- }
- else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
-
- // both are range partitioned, check that partitioning fields are equivalently chosen
- return checkEquivalentFieldPositionsInKeyFields(
- produced1.getPartitioningFields(), produced2.getPartitioningFields());
-
- }
- else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
- produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {
-
- // both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
- return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
- produced2.getPartitioningFields().isExactMatch(this.keys2) &&
- produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
- produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());
-
- }
- else {
-
- // no other partitioning valid, incl. ANY_PARTITIONING.
- // For co-groups we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
- return false;
- }
-
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- int numRelevantFields = this.keys1.size();
-
- Ordering prod1 = produced1.getOrdering();
- Ordering prod2 = produced2.getOrdering();
-
- if (prod1 == null || prod2 == null) {
- throw new CompilerException("The given properties do not meet this operators requirements.");
- }
-
- // check that order of fields is equivalent
- if (!checkEquivalentFieldPositionsInKeyFields(
- prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
- return false;
- }
-
- // check that order directions are equivalent
- for (int i = 0; i < numRelevantFields; i++) {
- if (prod1.getOrder(i) != prod2.getOrder(i)) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections();
-
- if (inputOrders == null || inputOrders.length < this.keys1.size()) {
- throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator.");
- } else if (inputOrders.length > this.keys1.size()) {
- boolean[] tmp = new boolean[this.keys1.size()];
- System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
- inputOrders = tmp;
- }
-
- return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- GlobalProperties gp = GlobalProperties.combine(in1, in2);
- if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
- gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gp.clearUniqueFieldCombinations();
- return gp;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- LocalProperties comb = LocalProperties.combine(in1, in2);
- return comb.clearUniqueFieldSets();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
deleted file mode 100644
index 8e7edeb..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class CoGroupWithSolutionSetFirstDescriptor extends CoGroupDescriptor {
-
- public CoGroupWithSolutionSetFirstDescriptor(FieldList keys1, FieldList keys2) {
- super(keys1, keys2);
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- RequestedLocalProperties none = new RequestedLocalProperties();
- RequestedLocalProperties sort = new RequestedLocalProperties(Utils.createOrdering(this.keys2));
- return Collections.singletonList(new LocalPropertiesPair(none, sort));
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- boolean[] inputOrders = in2.getLocalProperties().getOrdering() == null ? null : in2.getLocalProperties().getOrdering().getFieldSortDirections();
-
- if (inputOrders == null || inputOrders.length < this.keys2.size()) {
- throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator.");
- } else if (inputOrders.length > this.keys2.size()) {
- boolean[] tmp = new boolean[this.keys2.size()];
- System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
- inputOrders = tmp;
- }
-
- return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- return true;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return in2;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
deleted file mode 100644
index 016eb79..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.util.Utils;
-
-/**
- *
- */
-public class CoGroupWithSolutionSetSecondDescriptor extends CoGroupDescriptor {
-
- public CoGroupWithSolutionSetSecondDescriptor(FieldList keys1, FieldList keys2) {
- super(keys1, keys2);
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- RequestedLocalProperties sort = new RequestedLocalProperties(Utils.createOrdering(this.keys1));
- RequestedLocalProperties none = new RequestedLocalProperties();
- return Collections.singletonList(new LocalPropertiesPair(sort, none));
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- return true;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return in1;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
deleted file mode 100644
index bcd4d73..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class CollectorMapDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.COLLECTOR_MAP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.COLLECTOR_MAP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setAnyDistribution();
- return Collections.singletonList(rgp);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
deleted file mode 100644
index 4a467dc..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class CrossBlockOuterFirstDescriptor extends CartesianProductDescriptor {
-
- public CrossBlockOuterFirstDescriptor() {
- this(true, true);
- }
-
- public CrossBlockOuterFirstDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
- super(allowBroadcastFirst, allowBroadcastSecond);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return new LocalProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
deleted file mode 100644
index 212e048..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class CrossBlockOuterSecondDescriptor extends CartesianProductDescriptor {
-
- public CrossBlockOuterSecondDescriptor() {
- this(true, true);
- }
-
- public CrossBlockOuterSecondDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
- super(allowBroadcastFirst, allowBroadcastSecond);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return new LocalProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
deleted file mode 100644
index 9c7bd63..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class CrossStreamOuterFirstDescriptor extends CartesianProductDescriptor {
-
- public CrossStreamOuterFirstDescriptor() {
- this(true, true);
- }
-
- public CrossStreamOuterFirstDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
- super(allowBroadcastFirst, allowBroadcastSecond);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- // uniqueness becomes grouping with streamed nested loops
- if ((in1.getGroupedFields() == null || in1.getGroupedFields().size() == 0) &&
- in1.getUniqueFields() != null && in1.getUniqueFields().size() > 0)
- {
- return LocalProperties.forGrouping(in1.getUniqueFields().iterator().next().toFieldList());
- } else {
- return in1.clearUniqueFieldSets();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
deleted file mode 100644
index 3fabad6..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class CrossStreamOuterSecondDescriptor extends CartesianProductDescriptor {
-
- public CrossStreamOuterSecondDescriptor() {
- this(true, true);
- }
-
- public CrossStreamOuterSecondDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
- super(allowBroadcastFirst, allowBroadcastSecond);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- // uniqueness becomes grouping with streamed nested loops
- if ((in2.getGroupedFields() == null || in2.getGroupedFields().size() == 0) &&
- in2.getUniqueFields() != null && in2.getUniqueFields().size() > 0)
- {
- return LocalProperties.forGrouping(in2.getUniqueFields().iterator().next().toFieldList());
- } else {
- return in2.clearUniqueFieldSets();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
deleted file mode 100644
index 81c823f..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class FilterDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.FLAT_MAP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Filter ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setAnyDistribution();
- return Collections.singletonList(rgp);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
deleted file mode 100644
index b915e45..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class FlatMapDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.FLAT_MAP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "FlatMap ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setAnyDistribution();
- return Collections.singletonList(rgp);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
deleted file mode 100644
index b648386..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The properties file belonging to the GroupCombineNode. It translates the GroupCombine operation
- * to the driver strategy SORTED_GROUP_COMBINE and sets the relevant grouping and sorting keys.
- * @see org.apache.flink.optimizer.dag.GroupCombineNode
- */
-public final class GroupCombineProperties extends OperatorDescriptorSingle {
-
- private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
-
- public GroupCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
- super(groupKeys);
-
- // if we have an additional ordering, construct the ordering to have primarily the grouping fields
-
- this.ordering = new Ordering();
- for (Integer key : this.keyList) {
- this.ordering.appendOrdering(key, null, Order.ANY);
- }
-
- // and next the additional order fields
- if (additionalOrderKeys != null) {
- for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
- Integer field = additionalOrderKeys.getFieldNumber(i);
- Order order = additionalOrderKeys.getOrder(i);
- this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
- }
- }
-
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.SORTED_GROUP_COMBINE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- node.setDegreeOfParallelism(in.getSource().getParallelism());
-
- // sorting key info
- SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode(
- node,
- "GroupCombine (" + node.getOperator().getName() + ")",
- in, // reuse the combine strategy also used in the group reduce
- DriverStrategy.SORTED_GROUP_COMBINE, this.keyList);
-
- // set sorting comparator key info
- singleInputPlanNode.setDriverKeyInfo(this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections(), 0);
- // set grouping comparator key info
- singleInputPlanNode.setDriverKeyInfo(this.keyList, 1);
-
- return singleInputPlanNode;
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties props = new RequestedGlobalProperties();
- props.setRandomPartitioning();
- return Collections.singletonList(props);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED) {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
deleted file mode 100644
index ebd09f2..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-public final class GroupReduceProperties extends OperatorDescriptorSingle {
-
- private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
-
- private final Partitioner<?> customPartitioner;
-
-
- public GroupReduceProperties(FieldSet keys) {
- this(keys, null, null);
- }
-
- public GroupReduceProperties(FieldSet keys, Ordering additionalOrderKeys) {
- this(keys, additionalOrderKeys, null);
- }
-
- public GroupReduceProperties(FieldSet keys, Partitioner<?> customPartitioner) {
- this(keys, null, customPartitioner);
- }
-
- public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
- super(groupKeys);
-
- // if we have an additional ordering, construct the ordering to have primarily the grouping fields
- if (additionalOrderKeys != null) {
- this.ordering = new Ordering();
- for (Integer key : this.keyList) {
- this.ordering.appendOrdering(key, null, Order.ANY);
- }
-
- // and next the additional order fields
- for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
- Integer field = additionalOrderKeys.getFieldNumber(i);
- Order order = additionalOrderKeys.getOrder(i);
- this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
- }
- }
- else {
- this.ordering = null;
- }
-
- this.customPartitioner = customPartitioner;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.SORTED_GROUP_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties props = new RequestedGlobalProperties();
-
- if (customPartitioner == null) {
- props.setAnyPartitioning(this.keys);
- } else {
- props.setCustomPartitioned(this.keys, this.customPartitioner);
- }
- return Collections.singletonList(props);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- RequestedLocalProperties props = new RequestedLocalProperties();
- if (this.ordering == null) {
- props.setGroupedFields(this.keys);
- } else {
- props.setOrdering(this.ordering);
- }
- return Collections.singletonList(props);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
deleted file mode 100644
index c4f47d3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-
-public final class GroupReduceWithCombineProperties extends OperatorDescriptorSingle {
-
- private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
-
- private final Partitioner<?> customPartitioner;
-
-
- public GroupReduceWithCombineProperties(FieldSet groupKeys) {
- this(groupKeys, null, null);
- }
-
- public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
- this(groupKeys, additionalOrderKeys, null);
- }
-
- public GroupReduceWithCombineProperties(FieldSet groupKeys, Partitioner<?> customPartitioner) {
- this(groupKeys, null, customPartitioner);
- }
-
- public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
- super(groupKeys);
-
- // if we have an additional ordering, construct the ordering to have primarily the grouping fields
- if (additionalOrderKeys != null) {
- this.ordering = new Ordering();
- for (Integer key : this.keyList) {
- this.ordering.appendOrdering(key, null, Order.ANY);
- }
-
- // and next the additional order fields
- for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
- Integer field = additionalOrderKeys.getFieldNumber(i);
- Order order = additionalOrderKeys.getOrder(i);
- this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
- }
- } else {
- this.ordering = null;
- }
-
- this.customPartitioner = customPartitioner;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.SORTED_GROUP_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
- // adjust a sort (changes grouping, so it must be for this driver to combining sort
- if (in.getLocalStrategy() == LocalStrategy.SORT) {
- if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
- throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
- }
- in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
- in.getLocalStrategySortOrder());
- }
- return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", in,
- DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
- } else {
- // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
- Channel toCombiner = new Channel(in.getSource());
- toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- // create an input node for combine with same DOP as input node
- GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
-
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator()
- .getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
- combiner.setCosts(new Costs(0, 0));
- combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
- // set sorting comparator key info
- combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
- // set grouping comparator key info
- combiner.setDriverKeyInfo(this.keyList, 1);
-
- Channel toReducer = new Channel(combiner);
- toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
- in.getShipStrategySortOrder(), in.getDataExchangeMode());
- toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
- in.getLocalStrategySortOrder());
-
- return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
- toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
- }
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties props = new RequestedGlobalProperties();
- if (customPartitioner == null) {
- props.setAnyPartitioning(this.keys);
- } else {
- props.setCustomPartitioned(this.keys, this.customPartitioner);
- }
- return Collections.singletonList(props);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- RequestedLocalProperties props = new RequestedLocalProperties();
- if (this.ordering == null) {
- props.setGroupedFields(this.keys);
- } else {
- props.setOrdering(this.ordering);
- }
- return Collections.singletonList(props);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
deleted file mode 100644
index fec72a9..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class HashJoinBuildFirstProperties extends AbstractJoinDescriptor {
-
- public HashJoinBuildFirstProperties(FieldList keys1, FieldList keys2) {
- super(keys1, keys2);
- }
-
- public HashJoinBuildFirstProperties(FieldList keys1, FieldList keys2,
- boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
- {
- super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.HYBRIDHASH_BUILD_FIRST;
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- // all properties are possible
- return Collections.singletonList(new LocalPropertiesPair(new RequestedLocalProperties(), new RequestedLocalProperties()));
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- DriverStrategy strategy;
-
- if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
- // sanity check that the first input is cached and remove that cache
- if (!in1.getTempMode().isCached()) {
- throw new CompilerException("No cache at point where static and dynamic parts meet.");
- }
- in1.setTempMode(in1.getTempMode().makeNonCached());
- strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
- }
- else {
- strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
- }
- return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return new LocalProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
deleted file mode 100644
index f9d1e6c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-public final class HashJoinBuildSecondProperties extends AbstractJoinDescriptor {
-
- public HashJoinBuildSecondProperties(FieldList keys1, FieldList keys2) {
- super(keys1, keys2);
- }
-
- public HashJoinBuildSecondProperties(FieldList keys1, FieldList keys2,
- boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
- {
- super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.HYBRIDHASH_BUILD_SECOND;
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- // all properties are possible
- return Collections.singletonList(new LocalPropertiesPair(
- new RequestedLocalProperties(), new RequestedLocalProperties()));
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- DriverStrategy strategy;
-
- if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
- // sanity check that the first input is cached and remove that cache
- if (!in2.getTempMode().isCached()) {
- throw new CompilerException("No cache at point where static and dynamic parts meet.");
- }
-
- in2.setTempMode(in2.getTempMode().makeNonCached());
- strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
- }
- else {
- strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
- }
- return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return new LocalProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
deleted file mode 100644
index 9f14d2a..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class MapDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.MAP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.MAP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setAnyDistribution();
- return Collections.singletonList(rgp);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
deleted file mode 100644
index 1489097..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class MapPartitionDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.MAP_PARTITION;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "MapPartition ("+node.getOperator().getName()+")", in, DriverStrategy.MAP_PARTITION);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setAnyDistribution();
- return Collections.singletonList(rgp);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps;
- }
-}
[30/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
deleted file mode 100644
index 2d8377e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
+++ /dev/null
@@ -1,579 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.postpass;
-
-import java.util.Map;
-
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerPostPassException;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-
-/**
- *
- */
-public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> implements OptimizerPostPass {
-
- private boolean propagateParentSchemaDown = true;
-
-
- public boolean isPropagateParentSchemaDown() {
- return propagateParentSchemaDown;
- }
-
- public void setPropagateParentSchemaDown(boolean propagateParentSchemaDown) {
- this.propagateParentSchemaDown = propagateParentSchemaDown;
- }
-
- // --------------------------------------------------------------------------------------------
- // Generic schema inferring traversal
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void postPass(OptimizedPlan plan) {
- for (SinkPlanNode sink : plan.getDataSinks()) {
- traverse(sink, null, true);
- }
- }
-
- @SuppressWarnings("unchecked")
- protected void traverse(PlanNode node, T parentSchema, boolean createUtilities) {
- // distinguish the node types
- if (node instanceof SinkPlanNode) {
- SinkPlanNode sn = (SinkPlanNode) node;
- Channel inchannel = sn.getInput();
-
- T schema = createEmptySchema();
- sn.postPassHelper = schema;
-
- // add the sinks information to the schema
- try {
- getSinkSchema(sn, schema);
- }
- catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Conflicting type infomation for the data sink '" +
- sn.getSinkNode().getOperator().getName() + "'.");
- }
-
- // descend to the input channel
- try {
- propagateToChannel(schema, inchannel, createUtilities);
- }
- catch (MissingFieldTypeInfoException ex) {
- throw new CompilerPostPassException("Missing type infomation for the channel that inputs to the data sink '" +
- sn.getSinkNode().getOperator().getName() + "'.");
- }
- }
- else if (node instanceof SourcePlanNode) {
- if (createUtilities) {
- ((SourcePlanNode) node).setSerializer(createSerializer(parentSchema, node));
- // nothing else to be done here. the source has no input and no strategy itself
- }
- }
- else if (node instanceof BulkIterationPlanNode) {
- BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
-
- // get the nodes current schema
- T schema;
- if (iterationNode.postPassHelper == null) {
- schema = createEmptySchema();
- iterationNode.postPassHelper = schema;
- } else {
- schema = (T) iterationNode.postPassHelper;
- }
- schema.increaseNumConnectionsThatContributed();
-
- // add the parent schema to the schema
- if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
- }
-
- // check whether all outgoing channels have not yet contributed. come back later if not.
- if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
- return;
- }
-
- if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
- }
-
- // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
- if (iterationNode.getRootOfTerminationCriterion() != null) {
- SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
- traverse(addMapper.getInput().getSource(), createEmptySchema(), false);
- try {
- addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
- } catch (MissingFieldTypeInfoException e) {
- throw new RuntimeException(e);
- }
- }
-
- // traverse the step function for the first time. create schema only, no utilities
- traverse(iterationNode.getRootOfStepFunction(), schema, false);
-
- T pss = (T) iterationNode.getPartialSolutionPlanNode().postPassHelper;
- if (pss == null) {
- throw new CompilerException("Error in Optimizer Post Pass: Partial solution schema is null after first traversal of the step function.");
- }
-
- // traverse the step function for the second time, taking the schema of the partial solution
- traverse(iterationNode.getRootOfStepFunction(), pss, createUtilities);
-
- if (iterationNode.getRootOfTerminationCriterion() != null) {
- SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
- traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities);
- try {
- addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
- } catch (MissingFieldTypeInfoException e) {
- throw new RuntimeException(e);
- }
- }
-
- // take the schema from the partial solution node and add its fields to the iteration result schema.
- // input and output schema need to be identical, so this is essentially a sanity check
- addSchemaToSchema(pss, schema, iterationNode.getProgramOperator().getName());
-
- // set the serializer
- if (createUtilities) {
- iterationNode.setSerializerForIterationChannel(createSerializer(pss, iterationNode.getPartialSolutionPlanNode()));
- }
-
- // done, we can now propagate our info down
- try {
- propagateToChannel(schema, iterationNode.getInput(), createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
- + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
- e.getFieldNumber());
- }
- }
- else if (node instanceof WorksetIterationPlanNode) {
- WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
-
- // get the nodes current schema
- T schema;
- if (iterationNode.postPassHelper == null) {
- schema = createEmptySchema();
- iterationNode.postPassHelper = schema;
- } else {
- schema = (T) iterationNode.postPassHelper;
- }
- schema.increaseNumConnectionsThatContributed();
-
- // add the parent schema to the schema (which refers to the solution set schema)
- if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
- }
-
- // check whether all outgoing channels have not yet contributed. come back later if not.
- if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
- return;
- }
- if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
- }
- if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
- }
-
- // traverse the step function
- // pass an empty schema to the next workset and the parent schema to the solution set delta
- // these first traversals are schema only
- traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false);
- traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false);
-
- T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper;
- T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper;
-
- if (wss == null) {
- throw new CompilerException("Error in Optimizer Post Pass: Workset schema is null after first traversal of the step function.");
- }
- if (sss == null) {
- throw new CompilerException("Error in Optimizer Post Pass: Solution set schema is null after first traversal of the step function.");
- }
-
- // make the second pass and instantiate the utilities
- traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities);
- traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities);
-
- // add the types from the solution set schema to the iteration's own schema. since
- // the solution set input and the result must have the same schema, this acts as a sanity check.
- try {
- for (Map.Entry<Integer, X> entry : sss) {
- Integer pos = entry.getKey();
- schema.addType(pos, entry.getValue());
- }
- } catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + iterationNode.getProgramOperator().getName() + "'. Contradicting types between the " +
- "result of the iteration and the solution set schema: " + e.getPreviousType() +
- " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
- }
-
- // set the serializers and comparators
- if (createUtilities) {
- WorksetIterationNode optNode = iterationNode.getIterationNode();
- iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
- iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
- try {
- iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
- } catch (MissingFieldTypeInfoException ex) {
- throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
- optNode.getOperator().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
- }
- }
-
- // done, we can now propagate our info down
- try {
- propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
- propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
- } catch (MissingFieldTypeInfoException ex) {
- throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
- + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
- ex.getFieldNumber());
- }
- }
- else if (node instanceof SingleInputPlanNode) {
- SingleInputPlanNode sn = (SingleInputPlanNode) node;
-
- // get the nodes current schema
- T schema;
- if (sn.postPassHelper == null) {
- schema = createEmptySchema();
- sn.postPassHelper = schema;
- } else {
- schema = (T) sn.postPassHelper;
- }
- schema.increaseNumConnectionsThatContributed();
- SingleInputNode optNode = sn.getSingleInputNode();
-
- // add the parent schema to the schema
- if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema, optNode, 0);
- }
-
- // check whether all outgoing channels have not yet contributed. come back later if not.
- if (schema.getNumConnectionsThatContributed() < sn.getOutgoingChannels().size()) {
- return;
- }
-
- // add the nodes local information
- try {
- getSingleInputNodeSchema(sn, schema);
- } catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
- }
-
- if (createUtilities) {
- // parameterize the node's driver strategy
- for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
- try {
- sn.setComparator(createComparator(sn.getKeys(i), sn.getSortOrders(i), schema),i);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getOperator().getName() + "'. Missing type information for key field " +
- e.getFieldNumber());
- }
- }
- }
-
- // done, we can now propagate our info down
- try {
- propagateToChannel(schema, sn.getInput(), createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" +
- optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
-
- // don't forget the broadcast inputs
- for (Channel c: sn.getBroadcastInputs()) {
- try {
- propagateToChannel(createEmptySchema(), c, createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
- optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
- }
- }
- else if (node instanceof DualInputPlanNode) {
- DualInputPlanNode dn = (DualInputPlanNode) node;
-
- // get the nodes current schema
- T schema1;
- T schema2;
- if (dn.postPassHelper1 == null) {
- schema1 = createEmptySchema();
- schema2 = createEmptySchema();
- dn.postPassHelper1 = schema1;
- dn.postPassHelper2 = schema2;
- } else {
- schema1 = (T) dn.postPassHelper1;
- schema2 = (T) dn.postPassHelper2;
- }
-
- schema1.increaseNumConnectionsThatContributed();
- schema2.increaseNumConnectionsThatContributed();
- TwoInputNode optNode = dn.getTwoInputNode();
-
- // add the parent schema to the schema
- if (propagateParentSchemaDown) {
- addSchemaToSchema(parentSchema, schema1, optNode, 0);
- addSchemaToSchema(parentSchema, schema2, optNode, 1);
- }
-
- // check whether all outgoing channels have not yet contributed. come back later if not.
- if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
- return;
- }
-
- // add the nodes local information
- try {
- getDualInputNodeSchema(dn, schema1, schema2);
- } catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
- }
-
- // parameterize the node's driver strategy
- if (createUtilities) {
- if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
- // set the individual comparators
- try {
- dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
- dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
-
- // set the pair comparator
- try {
- dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(),
- dn.getSortOrders(), schema1, schema2));
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
- optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
-
- }
- }
-
- // done, we can now propagate our info down
- try {
- propagateToChannel(schema1, dn.getInput1(), createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
- + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
- try {
- propagateToChannel(schema2, dn.getInput2(), createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
- + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
-
- // don't forget the broadcast inputs
- for (Channel c: dn.getBroadcastInputs()) {
- try {
- propagateToChannel(createEmptySchema(), c, createUtilities);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
- optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
- }
- }
- }
- else if (node instanceof NAryUnionPlanNode) {
- // only propagate the info down
- try {
- for (Channel channel : node.getInputs()) {
- propagateToChannel(parentSchema, channel, createUtilities);
- }
- } catch (MissingFieldTypeInfoException ex) {
- throw new CompilerPostPassException("Could not set up runtime strategy for the input channel to " +
- " a union node. Missing type information for field " + ex.getFieldNumber());
- }
- }
- // catch the sources of the iterative step functions
- else if (node instanceof BulkPartialSolutionPlanNode ||
- node instanceof SolutionSetPlanNode ||
- node instanceof WorksetPlanNode)
- {
- // get the nodes current schema
- T schema;
- String name;
- if (node instanceof BulkPartialSolutionPlanNode) {
- BulkPartialSolutionPlanNode psn = (BulkPartialSolutionPlanNode) node;
- if (psn.postPassHelper == null) {
- schema = createEmptySchema();
- psn.postPassHelper = schema;
- } else {
- schema = (T) psn.postPassHelper;
- }
- name = "partial solution of bulk iteration '" +
- psn.getPartialSolutionNode().getIterationNode().getOperator().getName() + "'";
- }
- else if (node instanceof SolutionSetPlanNode) {
- SolutionSetPlanNode ssn = (SolutionSetPlanNode) node;
- if (ssn.postPassHelper == null) {
- schema = createEmptySchema();
- ssn.postPassHelper = schema;
- } else {
- schema = (T) ssn.postPassHelper;
- }
- name = "solution set of workset iteration '" +
- ssn.getSolutionSetNode().getIterationNode().getOperator().getName() + "'";
- }
- else if (node instanceof WorksetPlanNode) {
- WorksetPlanNode wsn = (WorksetPlanNode) node;
- if (wsn.postPassHelper == null) {
- schema = createEmptySchema();
- wsn.postPassHelper = schema;
- } else {
- schema = (T) wsn.postPassHelper;
- }
- name = "workset of workset iteration '" +
- wsn.getWorksetNode().getIterationNode().getOperator().getName() + "'";
- } else {
- throw new CompilerException();
- }
-
- schema.increaseNumConnectionsThatContributed();
-
- // add the parent schema to the schema
- addSchemaToSchema(parentSchema, schema, name);
- }
- else {
- throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
- }
- }
-
- private void propagateToChannel(T schema, Channel channel, boolean createUtilities) throws MissingFieldTypeInfoException {
- if (createUtilities) {
- // the serializer always exists
- channel.setSerializer(createSerializer(schema));
-
- // parameterize the ship strategy
- if (channel.getShipStrategy().requiresComparator()) {
- channel.setShipStrategyComparator(
- createComparator(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder(), schema));
- }
-
- // parameterize the local strategy
- if (channel.getLocalStrategy().requiresComparator()) {
- channel.setLocalStrategyComparator(
- createComparator(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder(), schema));
- }
- }
-
- // propagate the channel's source model
- traverse(channel.getSource(), schema, createUtilities);
- }
-
- private void addSchemaToSchema(T sourceSchema, T targetSchema, String opName) {
- try {
- for (Map.Entry<Integer, X> entry : sourceSchema) {
- Integer pos = entry.getKey();
- targetSchema.addType(pos, entry.getValue());
- }
- } catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + opName + "' propagated from successor node. " +
- "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() +
- ". Most probable cause: Invalid constant field annotations.");
- }
- }
-
- private void addSchemaToSchema(T sourceSchema, T targetSchema, OptimizerNode optNode, int input) {
- try {
- for (Map.Entry<Integer, X> entry : sourceSchema) {
- Integer pos = entry.getKey();
- SemanticProperties sprops = optNode.getSemanticProperties();
-
- if (sprops != null && sprops.getForwardingTargetFields(input, pos) != null && sprops.getForwardingTargetFields(input, pos).contains(pos)) {
- targetSchema.addType(pos, entry.getValue());
- }
- }
- } catch (ConflictingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " +
- "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() +
- ". Most probable cause: Invalid constant field annotations.");
- }
- }
-
- private String getConflictingTypeErrorMessage(ConflictingFieldTypeInfoException e, String operatorName) {
- return "Conflicting type information for field " + e.getFieldNumber()
- + " in node '" + operatorName + "' between types declared in the node's "
- + "contract and types inferred from successor contracts. Conflicting types: "
- + e.getPreviousType() + " and " + e.getNewType()
- + ". Most probable cause: Invalid constant field annotations.";
- }
-
- private TypeSerializerFactory<?> createSerializer(T schema, PlanNode node) {
- try {
- return createSerializer(schema);
- } catch (MissingFieldTypeInfoException e) {
- throw new CompilerPostPassException("Missing type information while creating serializer for '" +
- node.getProgramOperator().getName() + "'.");
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Type specific methods that extract schema information
- // --------------------------------------------------------------------------------------------
-
- protected abstract T createEmptySchema();
-
- protected abstract void getSinkSchema(SinkPlanNode sink, T schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
-
- protected abstract void getSingleInputNodeSchema(SingleInputPlanNode node, T schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
-
- protected abstract void getDualInputNodeSchema(DualInputPlanNode node, T input1Schema, T input2Schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
-
- // --------------------------------------------------------------------------------------------
- // Methods to create serializers and comparators
- // --------------------------------------------------------------------------------------------
-
- protected abstract TypeSerializerFactory<?> createSerializer(T schema) throws MissingFieldTypeInfoException;
-
- protected abstract TypeComparatorFactory<?> createComparator(FieldList fields, boolean[] directions, T schema) throws MissingFieldTypeInfoException;
-
- protected abstract TypePairComparatorFactory<?, ?> createPairComparator(FieldList fields1, FieldList fields2, boolean[] sortDirections,
- T schema1, T schema2) throws MissingFieldTypeInfoException;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
deleted file mode 100644
index 5fdf3dd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.operators.DualInputOperator;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeinfo.AtomicType;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
-import org.apache.flink.api.java.typeutils.runtime.RuntimeComparatorFactory;
-import org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory;
-import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerPostPassException;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.optimizer.util.NoOpUnaryUdfOp;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- * The post-optimizer plan traversal. This traversal fills in the API specific utilities (serializers and
- * comparators).
- */
-public class JavaApiPostPass implements OptimizerPostPass {
-
- private final Set<PlanNode> alreadyDone = new HashSet<PlanNode>();
-
- private ExecutionConfig executionConfig = null;
-
- @Override
- public void postPass(OptimizedPlan plan) {
-
- executionConfig = plan.getOriginalPactPlan().getExecutionConfig();
-
- for (SinkPlanNode sink : plan.getDataSinks()) {
- traverse(sink);
- }
- }
-
-
- protected void traverse(PlanNode node) {
- if (!alreadyDone.add(node)) {
- // already worked on that one
- return;
- }
-
- // distinguish the node types
- if (node instanceof SinkPlanNode) {
- // descend to the input channel
- SinkPlanNode sn = (SinkPlanNode) node;
- Channel inchannel = sn.getInput();
- traverseChannel(inchannel);
- }
- else if (node instanceof SourcePlanNode) {
- TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
- ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
- }
- else if (node instanceof BulkIterationPlanNode) {
- BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
-
- if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
- }
-
- // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
- if (iterationNode.getRootOfTerminationCriterion() != null) {
- SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
- traverseChannel(addMapper.getInput());
- }
-
- BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
-
- // set the serializer
- iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
-
- // done, we can now propagate our info down
- traverseChannel(iterationNode.getInput());
- traverse(iterationNode.getRootOfStepFunction());
- }
- else if (node instanceof WorksetIterationPlanNode) {
- WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
-
- if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
- }
- if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
- throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
- }
-
- DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
-
- // set the serializers and comparators for the workset iteration
- iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
- iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
- iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
- iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
-
- // traverse the inputs
- traverseChannel(iterationNode.getInput1());
- traverseChannel(iterationNode.getInput2());
-
- // traverse the step function
- traverse(iterationNode.getSolutionSetDeltaPlanNode());
- traverse(iterationNode.getNextWorkSetPlanNode());
- }
- else if (node instanceof SingleInputPlanNode) {
- SingleInputPlanNode sn = (SingleInputPlanNode) node;
-
- if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
-
- // Special case for delta iterations
- if(sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
- traverseChannel(sn.getInput());
- return;
- } else {
- throw new RuntimeException("Wrong operator type found in post pass.");
- }
- }
-
- SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
-
- // parameterize the node's driver strategy
- for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
- sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i),
- getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
- }
- // done, we can now propagate our info down
- traverseChannel(sn.getInput());
-
- // don't forget the broadcast inputs
- for (Channel c: sn.getBroadcastInputs()) {
- traverseChannel(c);
- }
- }
- else if (node instanceof DualInputPlanNode) {
- DualInputPlanNode dn = (DualInputPlanNode) node;
-
- if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
- throw new RuntimeException("Wrong operator type found in post pass.");
- }
-
- DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
-
- // parameterize the node's driver strategy
- if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
- dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
- getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
- dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
- getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));
-
- dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
- dualInputOperator.getOperatorInfo().getSecondInputType()));
-
- }
-
- traverseChannel(dn.getInput1());
- traverseChannel(dn.getInput2());
-
- // don't forget the broadcast inputs
- for (Channel c: dn.getBroadcastInputs()) {
- traverseChannel(c);
- }
-
- }
- // catch the sources of the iterative step functions
- else if (node instanceof BulkPartialSolutionPlanNode ||
- node instanceof SolutionSetPlanNode ||
- node instanceof WorksetPlanNode)
- {
- // Do nothing :D
- }
- else if (node instanceof NAryUnionPlanNode){
- // Traverse to all child channels
- for (Channel channel : node.getInputs()) {
- traverseChannel(channel);
- }
- }
- else {
- throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
- }
- }
-
- private void traverseChannel(Channel channel) {
-
- PlanNode source = channel.getSource();
- Operator<?> javaOp = source.getProgramOperator();
-
-// if (!(javaOp instanceof BulkIteration) && !(javaOp instanceof JavaPlanNode)) {
-// throw new RuntimeException("Wrong operator type found in post pass: " + javaOp);
-// }
-
- TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
-
-
- if(javaOp instanceof GroupReduceOperatorBase &&
- (source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_REDUCE_COMBINE)) {
- GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp;
- type = groupNode.getInput().getOperatorInfo().getOutputType();
- }
- else if(javaOp instanceof PlanUnwrappingReduceGroupOperator &&
- source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) {
- PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp;
- type = groupNode.getInput().getOperatorInfo().getOutputType();
- }
-
- // the serializer always exists
- channel.setSerializer(createSerializer(type));
-
- // parameterize the ship strategy
- if (channel.getShipStrategy().requiresComparator()) {
- channel.setShipStrategyComparator(createComparator(type, channel.getShipStrategyKeys(),
- getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder())));
- }
-
- // parameterize the local strategy
- if (channel.getLocalStrategy().requiresComparator()) {
- channel.setLocalStrategyComparator(createComparator(type, channel.getLocalStrategyKeys(),
- getSortOrders(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder())));
- }
-
- // descend to the channel's source
- traverse(channel.getSource());
- }
-
-
- @SuppressWarnings("unchecked")
- private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) {
- Operator<?> op = node.getOptimizerNode().getOperator();
-
- if (op instanceof GenericDataSourceBase) {
- return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType();
- } else {
- throw new RuntimeException("Wrong operator type found in post pass.");
- }
- }
-
- private <T> TypeSerializerFactory<?> createSerializer(TypeInformation<T> typeInfo) {
- TypeSerializer<T> serializer = typeInfo.createSerializer(executionConfig);
-
- return new RuntimeSerializerFactory<T>(serializer, typeInfo.getTypeClass());
- }
-
- @SuppressWarnings("unchecked")
- private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
-
- TypeComparator<T> comparator;
- if (typeInfo instanceof CompositeType) {
- comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
- }
- else if (typeInfo instanceof AtomicType) {
- // handle grouping of atomic types
- comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
- }
- else {
- throw new RuntimeException("Unrecognized type: " + typeInfo);
- }
-
- return new RuntimeComparatorFactory<T>(comparator);
- }
-
- private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
- if (!(typeInfo1.isTupleType() || typeInfo1 instanceof PojoTypeInfo) && (typeInfo2.isTupleType() || typeInfo2 instanceof PojoTypeInfo)) {
- throw new RuntimeException("The runtime currently supports only keyed binary operations (such as joins) on tuples and POJO types.");
- }
-
-// @SuppressWarnings("unchecked")
-// TupleTypeInfo<T1> info1 = (TupleTypeInfo<T1>) typeInfo1;
-// @SuppressWarnings("unchecked")
-// TupleTypeInfo<T2> info2 = (TupleTypeInfo<T2>) typeInfo2;
-
- return new RuntimePairComparatorFactory<T1,T2>();
- }
-
- private static final boolean[] getSortOrders(FieldList keys, boolean[] orders) {
- if (orders == null) {
- orders = new boolean[keys.size()];
- Arrays.fill(orders, true);
- }
- return orders;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
deleted file mode 100644
index b9f6bfa..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-public final class MissingFieldTypeInfoException extends Exception {
-
- private static final long serialVersionUID = 8749941961302509358L;
-
- private final int fieldNumber;
-
- public MissingFieldTypeInfoException(int fieldNumber) {
- this.fieldNumber = fieldNumber;
- }
-
- public int getFieldNumber() {
- return fieldNumber;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
deleted file mode 100644
index ba0b7c7..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.postpass;
-
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-
-/**
- * Interface for visitors that process the optimizer's plan. Typical post processing applications are schema
- * finalization or the generation/parameterization of utilities for the actual data model.
- */
-public interface OptimizerPostPass {
-
- /**
- * Central post processing function. Invoked by the optimizer after the best plan has
- * been determined.
- *
- * @param plan The plan to be post processed.
- */
- void postPass(OptimizedPlan plan);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
deleted file mode 100644
index 1fc4c34..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.types.Key;
-
-
-public class PostPassUtils {
-
- public static <X> Class<? extends Key<?>>[] getKeys(AbstractSchema<Class< ? extends X>> schema, int[] fields) throws MissingFieldTypeInfoException {
- @SuppressWarnings("unchecked")
- Class<? extends Key<?>>[] keyTypes = new Class[fields.length];
-
- for (int i = 0; i < fields.length; i++) {
- Class<? extends X> type = schema.getType(fields[i]);
- if (type == null) {
- throw new MissingFieldTypeInfoException(i);
- } else if (Key.class.isAssignableFrom(type)) {
- @SuppressWarnings("unchecked")
- Class<? extends Key<?>> keyType = (Class<? extends Key<?>>) type;
- keyTypes[i] = keyType;
- } else {
- throw new CompilerException("The field type " + type.getName() +
- " cannot be used as a key because it does not implement the interface 'Key'");
- }
- }
-
- return keyTypes;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
deleted file mode 100644
index 8a2d006..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-import org.apache.flink.api.common.operators.DualInputOperator;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.RecordOperator;
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.api.common.typeutils.record.RecordComparatorFactory;
-import org.apache.flink.api.common.typeutils.record.RecordPairComparatorFactory;
-import org.apache.flink.api.common.typeutils.record.RecordSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerPostPassException;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.types.Key;
-
-/**
- * Post pass implementation for the Record data model. Does only type inference and creates
- * serializers and comparators.
- */
-public class RecordModelPostPass extends GenericFlatTypePostPass<Class<? extends Key<?>>, SparseKeySchema> {
-
- // --------------------------------------------------------------------------------------------
- // Type specific methods that extract schema information
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected SparseKeySchema createEmptySchema() {
- return new SparseKeySchema();
- }
-
- @Override
- protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException {
- GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getOperator();
- Ordering partitioning = sink.getPartitionOrdering();
- Ordering sorting = sink.getLocalOrder();
-
- try {
- if (partitioning != null) {
- addOrderingToSchema(partitioning, schema);
- }
- if (sorting != null) {
- addOrderingToSchema(sorting, schema);
- }
- } catch (ConflictingFieldTypeInfoException ex) {
- throw new CompilerPostPassException("Conflicting information found when adding data sink types. " +
- "Probable reason is contradicting type infos for partitioning and sorting ordering.");
- }
- }
-
- @Override
- protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema)
- throws CompilerPostPassException, ConflictingFieldTypeInfoException
- {
- // check that we got the right types
- SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getOperator();
- if (! (contract instanceof RecordOperator)) {
- throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation.");
- }
- RecordOperator recContract = (RecordOperator) contract;
-
- // add the information to the schema
- int[] localPositions = contract.getKeyColumns(0);
- Class<? extends Key<?>>[] types = recContract.getKeyClasses();
- for (int i = 0; i < localPositions.length; i++) {
- schema.addType(localPositions[i], types[i]);
- }
-
- // this is a temporary fix, we should solve this more generic
- if (contract instanceof GroupReduceOperatorBase) {
- Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder();
- if (groupOrder != null) {
- addOrderingToSchema(groupOrder, schema);
- }
- }
- }
-
- @Override
- protected void getDualInputNodeSchema(DualInputPlanNode node, SparseKeySchema input1Schema, SparseKeySchema input2Schema)
- throws CompilerPostPassException, ConflictingFieldTypeInfoException
- {
- // add the nodes local information. this automatically consistency checks
- DualInputOperator<?, ?, ?, ?> contract = node.getTwoInputNode().getOperator();
- if (! (contract instanceof RecordOperator)) {
- throw new CompilerPostPassException("Error: Operator is not a Pact Record based contract. Wrong compiler invokation.");
- }
-
- RecordOperator recContract = (RecordOperator) contract;
- int[] localPositions1 = contract.getKeyColumns(0);
- int[] localPositions2 = contract.getKeyColumns(1);
- Class<? extends Key<?>>[] types = recContract.getKeyClasses();
-
- if (localPositions1.length != localPositions2.length) {
- throw new CompilerException("Error: The keys for the first and second input have a different number of fields.");
- }
-
- for (int i = 0; i < localPositions1.length; i++) {
- input1Schema.addType(localPositions1[i], types[i]);
- }
- for (int i = 0; i < localPositions2.length; i++) {
- input2Schema.addType(localPositions2[i], types[i]);
- }
-
-
- // this is a temporary fix, we should solve this more generic
- if (contract instanceof CoGroupOperatorBase) {
- Ordering groupOrder1 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputOne();
- Ordering groupOrder2 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputTwo();
-
- if (groupOrder1 != null) {
- addOrderingToSchema(groupOrder1, input1Schema);
- }
- if (groupOrder2 != null) {
- addOrderingToSchema(groupOrder2, input2Schema);
- }
- }
- }
-
- private void addOrderingToSchema(Ordering o, SparseKeySchema schema) throws ConflictingFieldTypeInfoException {
- for (int i = 0; i < o.getNumberOfFields(); i++) {
- Integer pos = o.getFieldNumber(i);
- Class<? extends Key<?>> type = o.getType(i);
- schema.addType(pos, type);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Methods to create serializers and comparators
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected TypeSerializerFactory<?> createSerializer(SparseKeySchema schema) {
- return RecordSerializerFactory.get();
- }
-
- @Override
- protected RecordComparatorFactory createComparator(FieldList fields, boolean[] directions, SparseKeySchema schema)
- throws MissingFieldTypeInfoException
- {
- int[] positions = fields.toArray();
- Class<? extends Key<?>>[] keyTypes = PostPassUtils.getKeys(schema, positions);
- return new RecordComparatorFactory(positions, keyTypes, directions);
- }
-
- @Override
- protected RecordPairComparatorFactory createPairComparator(FieldList fields1, FieldList fields2, boolean[] sortDirections,
- SparseKeySchema schema1, SparseKeySchema schema2)
- {
- return RecordPairComparatorFactory.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
deleted file mode 100644
index e14888e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.flink.types.Key;
-
-/**
- * Class encapsulating a schema map (int column position -> column type) and a reference counter.
- */
-public class SparseKeySchema extends AbstractSchema<Class<? extends Key<?>>> {
-
- private final Map<Integer, Class<? extends Key<?>>> schema;
-
-
- public SparseKeySchema() {
- this.schema = new HashMap<Integer, Class<? extends Key<?>>>();
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void addType(int key, Class<? extends Key<?>> type) throws ConflictingFieldTypeInfoException {
- Class<? extends Key<?>> previous = this.schema.put(key, type);
- if (previous != null && previous != type) {
- throw new ConflictingFieldTypeInfoException(key, previous, type);
- }
- }
-
- @Override
- public Class<? extends Key<?>> getType(int field) {
- return this.schema.get(field);
- }
-
- @Override
- public Iterator<Entry<Integer, Class<? extends Key<?>>>> iterator() {
- return this.schema.entrySet().iterator();
- }
-
- public int getNumTypes() {
- return this.schema.size();
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- return this.schema.hashCode() ^ getNumConnectionsThatContributed();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof SparseKeySchema) {
- SparseKeySchema other = (SparseKeySchema) obj;
- return this.schema.equals(other.schema) &&
- this.getNumConnectionsThatContributed() == other.getNumConnectionsThatContributed();
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "<" + getNumConnectionsThatContributed() + "> : " + this.schema.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
deleted file mode 100644
index bd35b5c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.util.Visitor;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A traversal that collects cascading binary unions into a single n-ary
- * union operator. The exception is, when on of the union inputs is materialized, such as in the
- * static-code-path-cache in iterations.
- */
-public class BinaryUnionReplacer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (this.seenBefore.add(visitable)) {
- if (visitable instanceof IterationPlanNode) {
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
- }
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
-
- if (visitable instanceof BinaryUnionPlanNode) {
-
- final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- final Channel in1 = unionNode.getInput1();
- final Channel in2 = unionNode.getInput2();
-
- if (!unionNode.unionsStaticAndDynamicPath()) {
-
- // both on static path, or both on dynamic path. we can collapse them
- NAryUnionPlanNode newUnionNode;
-
- List<Channel> inputs = new ArrayList<Channel>();
- collect(in1, inputs);
- collect(in2, inputs);
-
- newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
- unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
-
- newUnionNode.setParallelism(unionNode.getParallelism());
-
- for (Channel c : inputs) {
- c.setTarget(newUnionNode);
- }
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- else {
- // union between the static and the dynamic path. we need to handle this for now
- // through a special union operator
-
- // make sure that the first input is the cached (static) and the second input is the dynamic
- if (in1.isOnDynamicPath()) {
- BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
-
- in1.setTarget(newUnionNode);
- in2.setTarget(newUnionNode);
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- }
- }
- }
-
- public void collect(Channel in, List<Channel> inputs) {
- if (in.getSource() instanceof NAryUnionPlanNode) {
- // sanity check
- if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
- throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
- }
- if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
- throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
- }
-
- inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
- } else {
- // is not a collapsed union node, so we take the channel directly
- inputs.add(in);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
deleted file mode 100644
index 4730546..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.util.Visitor;
-
-/**
- * This traversal of the optimizer DAG computes the information needed to track
- * branches and joins in the data flow. This is important to support plans
- * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
- * output into more than one other node).
- */
-public final class BranchesVisitor implements Visitor<OptimizerNode> {
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- return node.getOpenBranches() == null;
- }
-
- @Override
- public void postVisit(OptimizerNode node) {
- if (node instanceof IterationNode) {
- ((IterationNode) node).acceptForStepFunction(this);
- }
-
- node.computeUnclosedBranchStack();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
deleted file mode 100644
index 160ef95..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Union;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.base.CrossOperatorBase;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.api.common.operators.base.FilterOperatorBase;
-import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dag.BulkIterationNode;
-import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
-import org.apache.flink.optimizer.dag.CoGroupNode;
-import org.apache.flink.optimizer.dag.CollectorMapNode;
-import org.apache.flink.optimizer.dag.CrossNode;
-import org.apache.flink.optimizer.dag.DagConnection;
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dag.FilterNode;
-import org.apache.flink.optimizer.dag.FlatMapNode;
-import org.apache.flink.optimizer.dag.GroupCombineNode;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.JoinNode;
-import org.apache.flink.optimizer.dag.MapNode;
-import org.apache.flink.optimizer.dag.MapPartitionNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.PartitionNode;
-import org.apache.flink.optimizer.dag.ReduceNode;
-import org.apache.flink.optimizer.dag.SolutionSetNode;
-import org.apache.flink.optimizer.dag.SortPartitionNode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.optimizer.dag.WorksetNode;
-import org.apache.flink.util.Visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This traversal creates the optimizer DAG from a program.
- * It works as a visitor that walks the program's flow in a depth-first fashion, starting from the data sinks.
- * During the descend, it creates an optimizer node for each operator, respectively data source or -sink.
- * During the ascend, it connects the nodes to the full graph.
- */
-public class GraphCreatingVisitor implements Visitor<Operator<?>> {
-
- private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
- // corresponding optimizer nodes
-
- private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
-
- private final int defaultParallelism; // the default degree of parallelism
-
- private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
-
- private final ExecutionMode defaultDataExchangeMode;
-
- private final boolean forceDOP;
-
-
- public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
- this(null, false, defaultParallelism, defaultDataExchangeMode, null);
- }
-
- private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
- ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
- if (closure == null){
- con2node = new HashMap<Operator<?>, OptimizerNode>();
- } else {
- con2node = closure;
- }
-
- this.sinks = new ArrayList<DataSinkNode>(2);
- this.defaultParallelism = defaultParallelism;
- this.parent = parent;
- this.defaultDataExchangeMode = dataExchangeMode;
- this.forceDOP = forceDOP;
- }
-
- public List<DataSinkNode> getSinks() {
- return sinks;
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public boolean preVisit(Operator<?> c) {
- // check if we have been here before
- if (this.con2node.containsKey(c)) {
- return false;
- }
-
- final OptimizerNode n;
-
- // create a node for the operator (or sink or source) if we have not been here before
- if (c instanceof GenericDataSinkBase) {
- DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
- this.sinks.add(dsn);
- n = dsn;
- }
- else if (c instanceof GenericDataSourceBase) {
- n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
- }
- else if (c instanceof MapOperatorBase) {
- n = new MapNode((MapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof MapPartitionOperatorBase) {
- n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
- n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FlatMapOperatorBase) {
- n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FilterOperatorBase) {
- n = new FilterNode((FilterOperatorBase<?, ?>) c);
- }
- else if (c instanceof ReduceOperatorBase) {
- n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
- }
- else if (c instanceof GroupCombineOperatorBase) {
- n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof GroupReduceOperatorBase) {
- n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof JoinOperatorBase) {
- n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CoGroupOperatorBase) {
- n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CrossOperatorBase) {
- n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof BulkIterationBase) {
- n = new BulkIterationNode((BulkIterationBase<?>) c);
- }
- else if (c instanceof DeltaIterationBase) {
- n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
- }
- else if (c instanceof Union){
- n = new BinaryUnionNode((Union<?>) c);
- }
- else if (c instanceof PartitionOperatorBase) {
- n = new PartitionNode((PartitionOperatorBase<?>) c);
- }
- else if (c instanceof SortPartitionOperatorBase) {
- n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
- }
- else if (c instanceof BulkIterationBase.PartialSolutionPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final BulkIterationBase.PartialSolutionPlaceHolder<?> holder = (BulkIterationBase.PartialSolutionPlaceHolder<?>) c;
- final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
- final BulkIterationNode containingIterationNode =
- (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else if (c instanceof DeltaIterationBase.WorksetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final DeltaIterationBase.WorksetPlaceHolder<?> holder = (DeltaIterationBase.WorksetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- WorksetNode p = new WorksetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else if (c instanceof DeltaIterationBase.SolutionSetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final DeltaIterationBase.SolutionSetPlaceHolder<?> holder = (DeltaIterationBase.SolutionSetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else {
- throw new IllegalArgumentException("Unknown operator type: " + c);
- }
-
- this.con2node.put(c, n);
-
- // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
- // key-less reducer (all-reduce)
- if (n.getParallelism() < 1) {
- // set the degree of parallelism
- int par = c.getDegreeOfParallelism();
- if (par > 0) {
- if (this.forceDOP && par != this.defaultParallelism) {
- par = this.defaultParallelism;
- Optimizer.LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
- "currently fixed to the parallelism of the surrounding operator (the iteration).");
- }
- } else {
- par = this.defaultParallelism;
- }
- n.setDegreeOfParallelism(par);
- }
-
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> c) {
-
- OptimizerNode n = this.con2node.get(c);
-
- // first connect to the predecessors
- n.setInput(this.con2node, this.defaultDataExchangeMode);
- n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
-
- // if the node represents a bulk iteration, we recursively translate the data flow now
- if (n instanceof BulkIterationNode) {
- final BulkIterationNode iterNode = (BulkIterationNode) n;
- final BulkIterationBase<?> iter = iterNode.getIterationContract();
-
- // pass a copy of the no iterative part into the iteration translation,
- // in case the iteration references its closure
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
- iterNode.getParallelism(), defaultDataExchangeMode, closure);
-
- BulkPartialSolutionNode partialSolution;
-
- iter.getNextPartialSolution().accept(recursiveCreator);
-
- partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
- OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
- if (partialSolution == null) {
- throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
- }
-
-
- OptimizerNode terminationCriterion = null;
-
- if (iter.getTerminationCriterion() != null) {
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
-
- // no intermediate node yet, traverse from the termination criterion to build the missing parts
- if (terminationCriterion == null) {
- iter.getTerminationCriterion().accept(recursiveCreator);
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
- }
- }
-
- iterNode.setPartialSolution(partialSolution);
- iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(identifier);
- }
- else if (n instanceof WorksetIterationNode) {
- final WorksetIterationNode iterNode = (WorksetIterationNode) n;
- final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
-
- // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
- // One check is for free during the translation, we do the other check here as a pre-condition
- {
- StepFunctionValidator wsf = new StepFunctionValidator();
- iter.getNextWorkset().accept(wsf);
- if (!wsf.hasFoundWorkset()) {
- throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
- "This is a prerequisite in delta iterations.");
- }
- }
-
- // calculate the closure of the anonymous function
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
- this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure);
-
- // descend from the solution set delta. check that it depends on both the workset
- // and the solution set. If it does depend on both, this descend should create both nodes
- iter.getSolutionSetDelta().accept(recursiveCreator);
-
- final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
-
- if (worksetNode == null) {
- throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
- "This is a prerequisite in delta iterations.");
- }
-
- iter.getNextWorkset().accept(recursiveCreator);
-
- SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
-
- if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
- solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
- }
- else {
- for (DagConnection conn : solutionSetNode.getOutgoingConnections()) {
- OptimizerNode successor = conn.getTarget();
-
- if (successor.getClass() == JoinNode.class) {
- // find out which input to the match the solution set is
- JoinNode mn = (JoinNode) successor;
- if (mn.getFirstPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(0);
- } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else if (successor.getClass() == CoGroupNode.class) {
- CoGroupNode cg = (CoGroupNode) successor;
- if (cg.getFirstPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(0);
- } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else {
- throw new InvalidProgramException(
- "Error: The only operations allowed on the solution set are Join and CoGroup.");
- }
- }
- }
-
- final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
- final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
-
- // set the step function nodes to the iteration node
- iterNode.setPartialSolution(solutionSetNode, worksetNode);
- iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(pathIdentifier);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
deleted file mode 100644
index b5c09e5..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dag.DagConnection;
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.util.Visitor;
-
-/**
- * This traversal of the optimizer DAG assigns IDs to each node (in a pre-order fashion),
- * and calls each node to compute its estimates. The latter happens in the postVisit function,
- * where it is guaranteed that all predecessors have computed their estimates.
- */
-public class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
-
- private final DataStatistics statistics;
-
- private int id = 1;
-
- public IdAndEstimatesVisitor(DataStatistics statistics) {
- this.statistics = statistics;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return visitable.getId() == -1;
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- // the node ids
- visitable.initId(this.id++);
-
- // connections need to figure out their maximum path depths
- for (DagConnection conn : visitable.getIncomingConnections()) {
- conn.initMaxDepth();
- }
- for (DagConnection conn : visitable.getBroadcastConnections()) {
- conn.initMaxDepth();
- }
-
- // the estimates
- visitable.computeOutputEstimates(this.statistics);
-
- // if required, recurse into the step function
- if (visitable instanceof IterationNode) {
- ((IterationNode) visitable).acceptForStepFunction(this);
- }
- }
-}
[42/53] [abbrv] flink git commit: [optimizer] Migrate first set of
tests (branching plans) to new API
Posted by se...@apache.org.
[optimizer] Migrate first set of tests (branching plans) to new API
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c2db1812
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c2db1812
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c2db1812
Branch: refs/heads/master
Commit: c2db18120c53dc8712b08369f7ea5b93ace98c6b
Parents: a9150b3
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 16:14:50 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
.../optimizer/BranchingPlansCompilerTest.java | 414 ++++++++-----------
1 file changed, 180 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c2db1812/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
index ff0e004..916aa27 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer;
import static org.junit.Assert.*;
@@ -26,6 +25,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
+import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
import org.apache.flink.api.common.Plan;
@@ -73,39 +78,27 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
final int SINKS = 5;
try {
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
-
- // construct the plan
- final String out1Path = "file:///test/1";
- final String out2Path = "file:///test/2";
-
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
-
- MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
- MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
-
- FileDataSink[] sinkA = new FileDataSink[SINKS];
- FileDataSink[] sinkB = new FileDataSink[SINKS];
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Long> source = env.generateSequence(1, 10000);
+
+ DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedC = source.map(new IdentityMapper<Long>());
+
for (int sink = 0; sink < SINKS; sink++) {
- sinkA[sink] = new FileDataSink(DummyOutputFormat.class, out1Path, mapA, "Sink A:" + sink);
- sinks.add(sinkA[sink]);
-
- sinkB[sink] = new FileDataSink(DummyOutputFormat.class, out2Path, mapC, "Sink B:" + sink);
- sinks.add(sinkB[sink]);
+ mappedA.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
}
-
- // return the PACT plan
- Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");
-
+
+ Plan plan = env.createProgramPlan("Plans With Multiple Data Sinks");
OptimizedPlan oPlan = compileNoStats(plan);
-
- // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -122,57 +115,44 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
* (SINK A) (SINK B) (SINK C)
* </pre>
*/
+ @SuppressWarnings("unchecked")
@Test
public void testBranchingWithMultipleDataSinks2() {
try {
- // construct the plan
- final String out1Path = "file:///test/1";
- final String out2Path = "file:///test/2";
- final String out3Path = "file:///test/3";
-
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Long> source = env.generateSequence(1, 10000);
+
+ DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());
+
+ mappedB.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
+
+ Plan plan = env.createProgramPlan();
+ Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());
- MapOperator mapA = MapOperator.builder(IdentityMap.class).input(sourceA).name("Map A").build();
- MapOperator mapB = MapOperator.builder(IdentityMap.class).input(mapA).name("Map B").build();
- MapOperator mapC = MapOperator.builder(IdentityMap.class).input(mapA).name("Map C").build();
-
- FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, mapB, "Sink A");
- FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, mapC, "Sink B");
- FileDataSink sinkC = new FileDataSink(DummyOutputFormat.class, out3Path, mapC, "Sink C");
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sinkA);
- sinks.add(sinkB);
- sinks.add(sinkC);
-
- // return the PACT plan
- Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");
-
OptimizedPlan oPlan = compileNoStats(plan);
-
+
// ---------- check the optimizer plan ----------
-
+
// number of sinks
- Assert.assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());
-
- // sinks contain all sink paths
- Set<String> allSinks = new HashSet<String>();
- allSinks.add(out1Path);
- allSinks.add(out2Path);
- allSinks.add(out3Path);
-
- for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
- Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
+ assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());
+
+ // remove matching sinks to check relation
+ for (SinkPlanNode sink : oPlan.getDataSinks()) {
+ assertTrue(sinks.remove(sink.getProgramOperator()));
}
-
- // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
+ assertTrue(sinks.isEmpty());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -203,72 +183,64 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
@Test
public void testBranchingSourceMultipleTimes() {
try {
- // construct the plan
- FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
-
- JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceA)
- .input2(sourceA)
- .build();
- JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceA)
- .input2(mat1)
- .build();
- JoinOperator mat3 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceA)
- .input2(mat2)
- .build();
- JoinOperator mat4 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceA)
- .input2(mat3)
- .build();
- JoinOperator mat5 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceA)
- .input2(mat4)
- .build();
-
- MapOperator ma = MapOperator.builder(new IdentityMap()).input(sourceA).build();
-
- JoinOperator mat6 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(ma)
- .build();
- JoinOperator mat7 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(mat6)
- .build();
- JoinOperator mat8 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(mat7)
- .build();
- JoinOperator mat9 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(mat8)
- .build();
- JoinOperator mat10 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(mat9)
- .build();
-
- CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0, 0)
- .input1(mat5)
- .input2(mat10)
- .build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, co);
-
- // return the PACT plan
- Plan plan = new Plan(sink, "Branching Source Multiple Times");
-
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> mapped = source.map(
+ new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
+ return null;
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+
+ joined5.coGroup(joined10)
+ .where(1).equalTo(1)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
+
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
+
+ Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- //Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -294,73 +266,54 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
@Test
public void testBranchingWithMultipleDataSinks() {
try {
- // construct the plan
- final String out1Path = "file:///test/1";
- final String out2Path = "file:///test/2";
- final String out3Path = "file:///test/3";
-
- FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
- FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
- FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), IN_FILE);
-
- CoGroupOperator co = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(sourceA)
- .input2(sourceB)
- .build();
- MapOperator ma = MapOperator.builder(new IdentityMap()).input(co).build();
- JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceB)
- .input2(sourceC)
- .build();
- JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(ma)
- .input2(mat1)
- .build();
- ReduceOperator r = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(ma)
- .build();
- CrossOperator c = CrossOperator.builder(new DummyCrossStub())
- .input1(r)
- .input2(mat2)
- .build();
-
- FileDataSink sinkA = new FileDataSink(new DummyOutputFormat(), out1Path, c);
- FileDataSink sinkB = new FileDataSink(new DummyOutputFormat(), out2Path, mat2);
- FileDataSink sinkC = new FileDataSink(new DummyOutputFormat(), out3Path, mat2);
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sinkA);
- sinks.add(sinkB);
- sinks.add(sinkC);
-
- // return the PACT plan
- Plan plan = new Plan(sinks, "Branching Plans With Multiple Data Sinks");
-
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
+ .where(0).equalTo(1)
+ .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
+ @Override
+ public void coGroup(Iterable<Tuple2<Long, Long>> first,
+ Iterable<Tuple2<Long, Long>> second,
+ Collector<Tuple2<Long, Long>> out) {
+ }
+ })
+ .map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
+ .where(0).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
+ .where(1).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> reduced = mapped
+ .groupBy(1)
+ .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());
+
+ reduced.cross(joined2)
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
+
+ joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+ joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+
+ Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
-
- // ---------- check the optimizer plan ----------
-
- // number of sinks
- Assert.assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());
-
- // sinks contain all sink paths
- Set<String> allSinks = new HashSet<String>();
- allSinks.add(out1Path);
- allSinks.add(out2Path);
- allSinks.add(out3Path);
-
- for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
- Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
- }
-
- // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@@ -860,48 +813,37 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
*/
@Test
public void testIterationWithStaticInput() {
- FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");
-
- MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
- input(source).
- name("Identity mapped source").
- build();
-
- ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
- input(source).
- name("Identity reduce source").
- build();
-
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(mappedSource);
- iteration.setMaximumNumberOfIterations(10);
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
- JoinOperator nextPartialSolution = JoinOperator.builder(DummyMatchStub.class, IntValue.class, 0,0).
- input1(iteration.getPartialSolution()).
- input2(reducedSource).
- name("Next partial solution").
- build();
+ DataSet<Long> source = env.generateSequence(1, 1000000);
- iteration.setNextPartialSolution(nextPartialSolution);
+ DataSet<Long> mapped = source.map(new IdentityMapper<Long>());
- FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Iteration sink");
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink);
+ DataSet<Long> reduced = source.groupBy(new IdentityKeyExtractor<Long>()).reduce(new SelectOneReducer<Long>());
- Plan plan = new Plan(sinks);
+ IterativeDataSet<Long> iteration = mapped.iterate(10);
+ iteration.closeWith(
+ iteration.join(reduced)
+ .where(new IdentityKeyExtractor<Long>())
+ .equalTo(new IdentityKeyExtractor<Long>())
+ .with(new DummyFlatJoinFunction<Long>()))
+ .output(new DiscardingOutputFormat<Long>());
- try{
- compileNoStats(plan);
- }catch(Exception e){
+ compileNoStats(env.createProgramPlan());
+ }
+ catch(Exception e){
e.printStackTrace();
- Assert.fail(e.getMessage());
+ fail(e.getMessage());
}
}
@Test
public void testBranchingBroadcastVariable() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+ env.setDegreeOfParallelism(100);
+
DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
@@ -972,6 +914,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
@Test
public void testMultipleIterations() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
@@ -1000,7 +943,8 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
@Test
public void testMultipleIterationsWithClosueBCVars() {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+ env.setDegreeOfParallelism(100);
+
DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
IterativeDataSet<String> iteration1 = input.iterate(100);
@@ -1026,7 +970,8 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
public void testBranchesOnlyInBCVariables1() {
try{
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+ env.setDegreeOfParallelism(100);
+
DataSet<Long> input = env.generateSequence(1, 10);
DataSet<Long> bc_input = env.generateSequence(1, 10);
@@ -1048,7 +993,8 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
public void testBranchesOnlyInBCVariables2() {
try{
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+ env.setDegreeOfParallelism(100);
+
DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 10).map(new Duplicator<Long>()).name("proper input");
DataSet<Long> bc_input1 = env.generateSequence(1, 10).name("BC input 1");
[15/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
new file mode 100644
index 0000000..a9d14e8
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dataproperties;
+
+import java.util.Arrays;
+
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+
+/**
+ * This class represents the local properties of the data that are requested by an operator.
+ * Local properties are the properties within one partition.
+ * Operators request the local properties they need for correct execution. Here are some example local
+ * properties requested by certain operators:
+ * <ul>
+ * <li>"groupBy/reduce" will request the data to be grouped on the key fields.</li>
+ * <li>A sort-merge join will request the data from each input to be sorted on the respective join key.</li>
+ * </ul>
+ */
+public class RequestedLocalProperties implements Cloneable {
+
+ private Ordering ordering; // order inside a partition, null if not ordered
+
+ private FieldSet groupedFields; // fields by which the stream is grouped. null if not grouped.
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Default constructor for trivial local properties. No order, no grouping, no uniqueness.
+ */
+ public RequestedLocalProperties() {}
+
+ /**
+ * Creates interesting properties for the given ordering.
+ *
+ * @param ordering The interesting ordering.
+ */
+ public RequestedLocalProperties(Ordering ordering) {
+ this.ordering = ordering;
+ }
+
+ /**
+ * Creates interesting properties for the given grouping.
+ *
+ * @param groupedFields The set of fields whose grouping is interesting.
+ */
+ public RequestedLocalProperties(FieldSet groupedFields) {
+ this.groupedFields = groupedFields;
+ }
+
+ /**
+ * This constructor is used only for internal copy creation.
+ *
+ * @param ordering The ordering represented by these local properties.
+ * @param groupedFields The grouped fields for these local properties.
+ */
+ private RequestedLocalProperties(Ordering ordering, FieldSet groupedFields) {
+ this.ordering = ordering;
+ this.groupedFields = groupedFields;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the key order.
+ *
+ * @return The key order, or <code>null</code> if nothing is ordered.
+ */
+ public Ordering getOrdering() {
+ return ordering;
+ }
+
+ /**
+ * Sets the order for these interesting local properties.
+ *
+ * @param ordering The order to set.
+ */
+ public void setOrdering(Ordering ordering) {
+ this.ordering = ordering;
+ }
+
+ /**
+ * Gets the grouped fields.
+ *
+ * @return The grouped fields, or <code>null</code> if nothing is grouped.
+ */
+ public FieldSet getGroupedFields() {
+ return this.groupedFields;
+ }
+
+ /**
+ * Sets the fields that are grouped in these data properties.
+ *
+ * @param groupedFields The fields that are grouped in these data properties.
+ */
+ public void setGroupedFields(FieldSet groupedFields) {
+ this.groupedFields = groupedFields;
+ }
+
+ /**
+ * Checks, if the properties in this object are trivial, i.e. only standard values.
+ */
+ public boolean isTrivial() {
+ return ordering == null && this.groupedFields == null;
+ }
+
+ /**
+ * This method resets the local properties to a state where no properties are given.
+ */
+ public void reset() {
+ this.ordering = null;
+ this.groupedFields = null;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Filters these properties by what can be preserved by the given SemanticProperties when propagated down
+ * to the given input.
+ *
+ * @param props The SemanticProperties which define which fields are preserved.
+ * @param input The index of the operator's input.
+ * @return The filtered RequestedLocalProperties
+ */
+ public RequestedLocalProperties filterBySemanticProperties(SemanticProperties props, int input) {
+
+ // no semantic properties, all local properties are filtered
+ if (props == null) {
+ throw new NullPointerException("SemanticProperties may not be null.");
+ }
+
+ if (this.ordering != null) {
+ Ordering newOrdering = new Ordering();
+
+ for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
+ int targetField = this.ordering.getInvolvedIndexes().get(i);
+ int sourceField = props.getForwardingSourceField(input, targetField);
+ if (sourceField >= 0) {
+ newOrdering.appendOrdering(sourceField, this.ordering.getType(i), this.ordering.getOrder(i));
+ } else {
+ return null;
+ }
+ }
+ return new RequestedLocalProperties(newOrdering);
+
+ } else if (this.groupedFields != null) {
+ FieldSet newGrouping = new FieldSet();
+
+ // check, whether the local key grouping is preserved
+ for (Integer targetField : this.groupedFields) {
+ int sourceField = props.getForwardingSourceField(input, targetField);
+ if (sourceField >= 0) {
+ newGrouping = newGrouping.addField(sourceField);
+ } else {
+ return null;
+ }
+ }
+ return new RequestedLocalProperties(newGrouping);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Checks, if this set of properties, as interesting properties, is met by the given
+ * properties.
+ *
+ * @param other
+ * The properties for which to check whether they meet these properties.
+ * @return True, if the properties are met, false otherwise.
+ */
+ public boolean isMetBy(LocalProperties other) {
+ if (this.ordering != null) {
+ // we demand an ordering
+ return other.getOrdering() != null && this.ordering.isMetBy(other.getOrdering());
+ } else if (this.groupedFields != null) {
+ // check if the other fields are unique
+ if (other.getGroupedFields() != null && other.getGroupedFields().isValidUnorderedPrefix(this.groupedFields)) {
+ return true;
+ } else {
+ return other.areFieldsUnique(this.groupedFields);
+ }
+ } else {
+ return true;
+ }
+ }
+
+ /**
+ * Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
+ *
+ * @param channel The channel to parametrize.
+ */
+ public void parameterizeChannel(Channel channel) {
+ LocalProperties current = channel.getLocalProperties();
+
+ if (isMetBy(current)) {
+ // we are met, all is good
+ channel.setLocalStrategy(LocalStrategy.NONE);
+ }
+ else if (this.ordering != null) {
+ channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections());
+ }
+ else if (this.groupedFields != null) {
+ boolean[] dirs = new boolean[this.groupedFields.size()];
+ Arrays.fill(dirs, true);
+ channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs);
+ }
+ else {
+ channel.setLocalStrategy(LocalStrategy.NONE);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (this.ordering == null ? 0 : this.ordering.hashCode());
+ result = prime * result + (this.groupedFields == null ? 0 : this.groupedFields.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RequestedLocalProperties) {
+ final RequestedLocalProperties other = (RequestedLocalProperties) obj;
+ return (this.ordering == other.ordering || (this.ordering != null && this.ordering.equals(other.ordering))) &&
+ (this.groupedFields == other.groupedFields || (this.groupedFields != null && this.groupedFields.equals(other.groupedFields)));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Requested Local Properties [ordering=" + this.ordering + ", grouped=" + this.groupedFields + "]";
+ }
+
+ @Override
+ public RequestedLocalProperties clone() {
+ return new RequestedLocalProperties(this.ordering, this.groupedFields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
new file mode 100644
index 0000000..707142f
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.deadlockdetect;
+
+public class DeadlockEdge {
+
+ private DeadlockVertex destination;
+
+ public DeadlockEdge( DeadlockVertex d ){
+ destination = d;
+ }
+
+ public DeadlockVertex getDestination() {
+ return destination;
+ }
+
+ public void setDestination(DeadlockVertex destination) {
+ this.destination = destination;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
new file mode 100644
index 0000000..cae75ad
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.deadlockdetect;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Set;
+
+import org.apache.flink.optimizer.plan.PlanNode;
+
+public class DeadlockGraph {
+
+ public Set<DeadlockVertex> vertices;
+
+ public DeadlockGraph() {
+ this.vertices = new HashSet<DeadlockVertex>();
+ }
+
+ public Set<DeadlockVertex> vertices() {
+ return vertices;
+ }
+
+ public DeadlockVertex addVertex(PlanNode original) {
+
+ DeadlockVertex v = new DeadlockVertex(original);
+ this.vertices.add(v);
+ return v;
+ }
+
+ public void addEdge(PlanNode source, PlanNode destination) {
+
+ DeadlockVertex dest = null;
+ for(DeadlockVertex v : vertices) {
+ if(v.getOriginal().equals(destination)) {
+ dest = v;
+ }
+ }
+
+ for(DeadlockVertex v : vertices) {
+ if(v.getOriginal().equals(source)) {
+ v.addEdge(dest);
+ }
+ }
+
+ }
+
+ public long size() {
+ return vertices.size();
+ }
+
+ public String toString() {
+ StringBuilder out = new StringBuilder();
+ out.append("------------ GRAPH ------------\n");
+ for (DeadlockVertex n : vertices) {
+ out.append("Node " +n+"_\n");
+ for(DeadlockEdge a: n.getOutEdges()) {
+ out.append("\t->"+a.getDestination()+"\n");
+ }
+ out.append("\n");
+ }
+
+ return out.toString();
+ }
+
+ public boolean hasCycle() {
+
+ Collection <DeadlockVertex> vertexCollect = this.vertices();
+
+ Queue <DeadlockVertex> q; // Queue will store vertices that have in-degree of zero
+
+ // Calculate the in-degree of all vertices
+ for (DeadlockVertex v: vertexCollect) {
+ v.setInDegree(0);
+ }
+
+ for (DeadlockVertex v: vertexCollect) {
+ for(DeadlockEdge edge : v.getOutEdges()) {
+ edge.getDestination().setInDegree(edge.getDestination().getInDegree()+1);
+ }
+ }
+
+ // Find all vertices with in-degree == 0 and put in queue
+ q = new LinkedList<DeadlockVertex>();
+ for (DeadlockVertex v : vertexCollect) {
+ if (v.getInDegree() == 0) {
+ q.offer(v);
+ }
+ }
+
+ while (!q.isEmpty()) {
+
+ DeadlockVertex v = q.poll();
+ this.vertices.remove(v);
+
+ for (DeadlockEdge e: v.getOutEdges()) {
+
+ DeadlockVertex w = e.getDestination();
+ w.setInDegree(w.getInDegree() - 1);
+
+ if(w.getInDegree() == 0) {
+ q.offer(w);
+ }
+ }
+ }
+
+ if (!vertexCollect.isEmpty() ){
+ return true; //Cycle found
+ }
+
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
new file mode 100644
index 0000000..19ee6c5
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.deadlockdetect;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Certain pipelined flows may lead to deadlocks, in which case we need to make sure the pipelines are broken or made elastic enough to prevent that.
+ *
+ * This is only relevat to pipelined data flows where one operator has more than one consumers (successors in the flow).
+ *
+ * Most cases are caught by the general logic that deals with branching/joining flows. The following cases need additional checks:
+ *
+ * <build>
+ * (source1) ------ (join)
+ * \ / <probe>
+ * \ /
+ * \/
+ * /\
+ * / \
+ * / \ <probe>
+ * (source2) ------(join)
+ * <build>
+ *
+ * Since both sources pipeline their data into a build and a probe side, they get stalled by the back pressure from the probe side (which waits for the build side to complete) and never finish the build side.
+ *
+ * We can model dependencies of pipelined / materialized connections and do a reguar cyclic dependencies check to detect such situations. Pipelined connections have a dependency from sender to receiver, non-pipelined (fully dammed) connections have a dependency from receiver to sender.
+ *
+ */
+public class DeadlockPreventer implements Visitor<PlanNode> {
+
+ private DeadlockGraph g;
+
+ public DeadlockPreventer() {
+ this.g = new DeadlockGraph();
+ }
+
+ public void resolveDeadlocks(List<? extends PlanNode> sinks) {
+
+ for(PlanNode s : sinks) {
+ s.accept(this);
+ }
+ if(g.hasCycle()) {
+
+ // in the remaining plan is a cycle
+ for(DeadlockVertex v : g.vertices) {
+
+ // first strategy to fix -> swap build and probe side
+ if(v.getOriginal().getDriverStrategy().equals(DriverStrategy.HYBRIDHASH_BUILD_FIRST)) {
+
+ v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
+
+ if(hasDeadlock(sinks)) {
+ // Didn't fix anything -> revert
+ v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
+ }
+ else {
+ // deadlock resolved
+ break;
+ }
+ }
+
+ // other direction
+ if(v.getOriginal().getDriverStrategy().equals(DriverStrategy.HYBRIDHASH_BUILD_SECOND)) {
+
+ v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
+
+ if(hasDeadlock(sinks)) {
+ // Didn't fix anything -> revert
+ v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
+ }
+ else {
+ // deadlock resolved
+ break;
+ }
+ }
+ }
+
+ // switching build and probe side did not help -> pipeline breaker
+ for(DeadlockVertex v : g.vertices) {
+ if(v.getOriginal() instanceof DualInputPlanNode) {
+ DualInputPlanNode n = (DualInputPlanNode) v.getOriginal();
+
+ // what is the pipelined side? (other side should be a dam, otherwise operator could not be source of deadlock)
+ if(!(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())
+ && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
+ n.getInput1().setTempMode(n.getInput1().getTempMode().makePipelineBreaker());
+ }
+ else if( !(n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())
+ && (n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())) {
+ n.getInput2().setTempMode(n.getInput2().getTempMode().makePipelineBreaker());
+ }
+
+ // Deadlock resolved?
+ if(!hasDeadlock(sinks)) {
+ break;
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Creates new DeadlockGraph from plan and checks for cycles
+ */
+ public boolean hasDeadlock(List<? extends PlanNode> sinks) {
+ this.g = new DeadlockGraph();
+
+ for(PlanNode s : sinks) {
+ s.accept(this);
+ }
+
+ if(g.hasCycle()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+
+ g.addVertex(visitable);
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+ if(visitable instanceof SingleInputPlanNode) {
+ SingleInputPlanNode n = (SingleInputPlanNode) visitable;
+
+ if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput().getLocalStrategy().dams() || n.getInput().getTempMode().breaksPipeline()) {
+ g.addEdge(n, n.getPredecessor());
+ }
+ else {
+ g.addEdge(n.getPredecessor(), n);
+ }
+ }
+ else if(visitable instanceof DualInputPlanNode) {
+ DualInputPlanNode n = (DualInputPlanNode) visitable;
+
+ if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline()) {
+ g.addEdge(n, n.getInput1().getSource());
+ }
+ else {
+ g.addEdge(n.getInput1().getSource(), n);
+ }
+
+ if(!n.getDriverStrategy().equals(DriverStrategy.NONE) && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
+ g.addEdge(n, n.getInput2().getSource());
+ }
+ else {
+ g.addEdge(n.getInput2().getSource(), n);
+ }
+ }
+
+
+ // recursively fix iterations
+ if (visitable instanceof BulkIterationPlanNode) {
+
+ DeadlockPreventer dp = new DeadlockPreventer();
+ List<PlanNode> planSinks = new ArrayList<PlanNode>(1);
+
+ BulkIterationPlanNode pspn = (BulkIterationPlanNode) visitable;
+ planSinks.add(pspn.getRootOfStepFunction());
+
+ dp.resolveDeadlocks(planSinks);
+
+ }
+ else if (visitable instanceof WorksetIterationPlanNode) {
+
+ DeadlockPreventer dp = new DeadlockPreventer();
+ List<PlanNode> planSinks = new ArrayList<PlanNode>(2);
+
+ WorksetIterationPlanNode pspn = (WorksetIterationPlanNode) visitable;
+ planSinks.add(pspn.getSolutionSetDeltaPlanNode());
+ planSinks.add(pspn.getNextWorkSetPlanNode());
+
+ dp.resolveDeadlocks(planSinks);
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
new file mode 100644
index 0000000..f26763e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.deadlockdetect;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.optimizer.plan.PlanNode;
+
+public class DeadlockVertex {
+
+ private PlanNode original;
+
+ private List<DeadlockEdge> outEdges;
+
+ private int inDegree;
+
+ public DeadlockVertex( PlanNode original ) {
+ this.original = original;
+ outEdges = new LinkedList<DeadlockEdge>();
+ inDegree = 0;
+ }
+
+ public void addEdge(DeadlockVertex destination) {
+
+ // no duplicates
+ for(DeadlockEdge e : outEdges) {
+ if(e.getDestination().equals(destination)) {
+ return;
+ }
+ }
+
+ DeadlockEdge e = new DeadlockEdge(destination);
+ this.outEdges.add(e);
+ }
+
+ public boolean equals(Object o) {
+
+ if(!(o instanceof DeadlockVertex)) {
+ return false;
+ }
+
+ DeadlockVertex v = (DeadlockVertex) o;
+ if(v.getOriginal().equals(this.getOriginal())) {
+ return true;
+ }
+
+ return false;
+ }
+
+ public int hashCode() {
+ return this.original.hashCode();
+ }
+
+ public String toString() {
+ return original.toString();
+ }
+ public PlanNode getOriginal() {
+ return original;
+ }
+
+ public void setOriginal(PlanNode original) {
+ this.original = original;
+ }
+
+ public int getInDegree() {
+ return inDegree;
+ }
+
+ public void setInDegree(int inDegree) {
+ this.inDegree = inDegree;
+ }
+
+ public List<DeadlockEdge> getOutEdges() {
+ return outEdges;
+ }
+
+ public void setOutEdges(List<DeadlockEdge> outEdges) {
+ this.outEdges = outEdges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
new file mode 100644
index 0000000..2b634ae
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+
+/**
+ * Defines the possible global properties for a join.
+ */
+public abstract class AbstractJoinDescriptor extends OperatorDescriptorDual {
+
+ private final boolean broadcastFirstAllowed;
+ private final boolean broadcastSecondAllowed;
+ private final boolean repartitionAllowed;
+
+ private Partitioner<?> customPartitioner;
+
+ protected AbstractJoinDescriptor(FieldList keys1, FieldList keys2) {
+ this(keys1, keys2, true, true, true);
+ }
+
+ protected AbstractJoinDescriptor(FieldList keys1, FieldList keys2,
+ boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
+ {
+ super(keys1, keys2);
+
+ this.broadcastFirstAllowed = broadcastFirstAllowed;
+ this.broadcastSecondAllowed = broadcastSecondAllowed;
+ this.repartitionAllowed = repartitionAllowed;
+ }
+
+ public void setCustomPartitioner(Partitioner<?> partitioner) {
+ customPartitioner = partitioner;
+ }
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+ ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
+
+ if (repartitionAllowed) {
+ // partition both (hash or custom)
+ if (this.customPartitioner == null) {
+
+ // we accept compatible partitionings of any type
+ RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
+ RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
+ partitioned_left_any.setAnyPartitioning(this.keys1);
+ partitioned_right_any.setAnyPartitioning(this.keys2);
+ pairs.add(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any));
+
+ // add strict hash partitioning of both inputs on their full key sets
+ RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
+ RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
+ partitioned_left_hash.setHashPartitioned(this.keys1);
+ partitioned_right_hash.setHashPartitioned(this.keys2);
+ pairs.add(new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
+ }
+ else {
+ RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
+ partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);
+
+ RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
+ partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);
+
+ return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
+ }
+
+
+ RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
+ if (customPartitioner == null) {
+ partitioned1.setAnyPartitioning(this.keys1);
+ } else {
+ partitioned1.setCustomPartitioned(this.keys1, this.customPartitioner);
+ }
+
+ RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
+ if (customPartitioner == null) {
+ partitioned2.setAnyPartitioning(this.keys2);
+ } else {
+ partitioned2.setCustomPartitioned(this.keys2, this.customPartitioner);
+ }
+
+ pairs.add(new GlobalPropertiesPair(partitioned1, partitioned2));
+ }
+
+ if (broadcastSecondAllowed) {
+ // replicate second
+ RequestedGlobalProperties any1 = new RequestedGlobalProperties();
+ RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
+ replicated2.setFullyReplicated();
+ pairs.add(new GlobalPropertiesPair(any1, replicated2));
+ }
+
+ if (broadcastFirstAllowed) {
+ // replicate first
+ RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
+ replicated1.setFullyReplicated();
+ RequestedGlobalProperties any2 = new RequestedGlobalProperties();
+ pairs.add(new GlobalPropertiesPair(replicated1, any2));
+ }
+ return pairs;
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2)
+ {
+ if (requested1.getPartitioning().isPartitionedOnKey() && requested2.getPartitioning().isPartitionedOnKey()) {
+
+ if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
+
+ // both are hash partitioned, check that partitioning fields are equivalently chosen
+ return checkEquivalentFieldPositionsInKeyFields(
+ produced1.getPartitioningFields(), produced2.getPartitioningFields());
+
+ }
+ else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+ produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
+
+ // both are range partitioned, check that partitioning fields are equivalently chosen
+ return checkEquivalentFieldPositionsInKeyFields(
+ produced1.getPartitioningFields(), produced2.getPartitioningFields());
+
+ }
+ else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
+ produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {
+
+ // both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
+ return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
+ produced2.getPartitioningFields().isExactMatch(this.keys2) &&
+ produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
+ produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());
+
+ }
+ else {
+
+ // no other partitioning valid, incl. ANY_PARTITIONING.
+ // For joins we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
+ return false;
+ }
+
+ } else {
+ return true;
+ }
+
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ GlobalProperties gp = GlobalProperties.combine(in1, in2);
+ if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
+ gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gp.clearUniqueFieldCombinations();
+ return gp;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
new file mode 100644
index 0000000..8125556
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ * The root of all descriptions of an operator to the optimizer.
+ */
+public interface AbstractOperatorDescriptor {
+
+ /**
+ * Gets the runtime strategy to be executed by this operator.
+ *
+ * @return The runtime strategy.
+ */
+ DriverStrategy getStrategy();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
new file mode 100644
index 0000000..9ee56b0
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+import java.util.Collections;
+import java.util.List;
+
+public final class AllGroupCombineProperties extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.ALL_GROUP_COMBINE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "GroupCombine ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_COMBINE);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
new file mode 100644
index 0000000..9efd8c7
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+public final class AllGroupReduceProperties extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.ALL_GROUP_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
new file mode 100644
index 0000000..b3c083a
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+public final class AllGroupWithPartialPreGroupProperties extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.ALL_GROUP_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
+ // locally connected, directly instantiate
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
+ in, DriverStrategy.ALL_GROUP_REDUCE);
+ } else {
+ // non forward case.plug in a combiner
+ Channel toCombiner = new Channel(in.getSource());
+ toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ // create an input node for combine with same DOP as input node
+ GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
+ "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
+ combiner.setCosts(new Costs(0, 0));
+ combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
+
+ Channel toReducer = new Channel(combiner);
+ toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
+ in.getShipStrategySortOrder(), in.getDataExchangeMode());
+
+ toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
+ toReducer, DriverStrategy.ALL_GROUP_REDUCE);
+ }
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
new file mode 100644
index 0000000..a172a60
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.ReduceNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+public final class AllReduceProperties extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.ALL_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
+ // locally connected, directly instantiate
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
+ in, DriverStrategy.ALL_REDUCE);
+ } else {
+ // non forward case.plug in a combiner
+ Channel toCombiner = new Channel(in.getSource());
+ toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ // create an input node for combine with same DOP as input node
+ ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
+ "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
+ combiner.setCosts(new Costs(0, 0));
+ combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
+
+ Channel toReducer = new Channel(combiner);
+ toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
+ in.getShipStrategySortOrder(), in.getDataExchangeMode());
+ toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
+ in.getLocalStrategySortOrder());
+
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
+ toReducer, DriverStrategy.ALL_REDUCE);
+ }
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return new GlobalProperties();
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return new LocalProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
new file mode 100644
index 0000000..8cc517e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/BinaryUnionOpDescriptor.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class BinaryUnionOpDescriptor extends OperatorDescriptorDual {
+
+ public BinaryUnionOpDescriptor() {
+ super();
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.UNION;
+ }
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ return new BinaryUnionPlanNode((BinaryUnionNode) node, in1, in2);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ GlobalProperties newProps = new GlobalProperties();
+
+ if (in1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ in2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ in1.getPartitioningFields().equals(in2.getPartitioningFields()))
+ {
+ newProps.setHashPartitioned(in1.getPartitioningFields());
+ }
+
+ return newProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ // all local properties are destroyed
+ return new LocalProperties();
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
new file mode 100644
index 0000000..f48e297
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+
+
+public abstract class CartesianProductDescriptor extends OperatorDescriptorDual {
+
+ private final boolean allowBroadcastFirst;
+ private final boolean allowBroadcastSecond;
+
+
+ protected CartesianProductDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
+ if (!(allowBroadcastFirst | allowBroadcastSecond)) {
+ throw new IllegalArgumentException();
+ }
+
+ this.allowBroadcastFirst = allowBroadcastFirst;
+ this.allowBroadcastSecond = allowBroadcastSecond;
+ }
+
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+ ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
+
+ if (this.allowBroadcastFirst) {
+ // replicate first
+ RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
+ replicated1.setFullyReplicated();
+ RequestedGlobalProperties any2 = new RequestedGlobalProperties();
+ pairs.add(new GlobalPropertiesPair(replicated1, any2));
+ }
+
+ if (this.allowBroadcastSecond) {
+ // replicate second
+ RequestedGlobalProperties any1 = new RequestedGlobalProperties();
+ RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
+ replicated2.setFullyReplicated();
+ pairs.add(new GlobalPropertiesPair(any1, replicated2));
+ }
+
+ return pairs;
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new LocalPropertiesPair(
+ new RequestedLocalProperties(), new RequestedLocalProperties()));
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ return new DualInputPlanNode(node, "Cross("+node.getOperator().getName()+")", in1, in2, getStrategy());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ GlobalProperties gp = GlobalProperties.combine(in1, in2);
+ if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
+ gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gp.clearUniqueFieldCombinations();
+ return gp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
new file mode 100644
index 0000000..368944e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+public class CoGroupDescriptor extends OperatorDescriptorDual {
+
+ private final Ordering ordering1; // ordering on the first input
+ private final Ordering ordering2; // ordering on the second input
+
+ private Partitioner<?> customPartitioner;
+
+
+ public CoGroupDescriptor(FieldList keys1, FieldList keys2) {
+ this(keys1, keys2, null, null);
+ }
+
+ public CoGroupDescriptor(FieldList keys1, FieldList keys2, Ordering additionalOrdering1, Ordering additionalOrdering2) {
+ super(keys1, keys2);
+
+ // if we have an additional ordering, construct the ordering to have primarily the grouping fields
+ if (additionalOrdering1 != null) {
+ this.ordering1 = new Ordering();
+ for (Integer key : this.keys1) {
+ this.ordering1.appendOrdering(key, null, Order.ANY);
+ }
+
+ // and next the additional order fields
+ for (int i = 0; i < additionalOrdering1.getNumberOfFields(); i++) {
+ Integer field = additionalOrdering1.getFieldNumber(i);
+ Order order = additionalOrdering1.getOrder(i);
+ this.ordering1.appendOrdering(field, additionalOrdering1.getType(i), order);
+ }
+ } else {
+ this.ordering1 = Utils.createOrdering(this.keys1);
+ }
+
+ // if we have an additional ordering, construct the ordering to have primarily the grouping fields
+ if (additionalOrdering2 != null) {
+ this.ordering2 = new Ordering();
+ for (Integer key : this.keys2) {
+ this.ordering2.appendOrdering(key, null, Order.ANY);
+ }
+
+ // and next the additional order fields
+ for (int i = 0; i < additionalOrdering2.getNumberOfFields(); i++) {
+ Integer field = additionalOrdering2.getFieldNumber(i);
+ Order order = additionalOrdering2.getOrder(i);
+ this.ordering2.appendOrdering(field, additionalOrdering2.getType(i), order);
+ }
+ } else {
+ this.ordering2 = Utils.createOrdering(this.keys2);
+ }
+ }
+
+ public void setCustomPartitioner(Partitioner<?> customPartitioner) {
+ this.customPartitioner = customPartitioner;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.CO_GROUP;
+ }
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+
+ if (this.customPartitioner == null) {
+
+ // we accept compatible partitionings of any type
+ RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
+ RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
+ partitioned_left_any.setAnyPartitioning(this.keys1);
+ partitioned_right_any.setAnyPartitioning(this.keys2);
+
+ // add strict hash partitioning of both inputs on their full key sets
+ RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
+ RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
+ partitioned_left_hash.setHashPartitioned(this.keys1);
+ partitioned_right_hash.setHashPartitioned(this.keys2);
+
+ return Arrays.asList(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any),
+ new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
+ }
+ else {
+ RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
+ partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);
+
+ RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
+ partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);
+
+ return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
+ }
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ RequestedLocalProperties sort1 = new RequestedLocalProperties(this.ordering1);
+ RequestedLocalProperties sort2 = new RequestedLocalProperties(this.ordering2);
+ return Collections.singletonList(new LocalPropertiesPair(sort1, sort2));
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2)
+ {
+
+ if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
+
+ // both are hash partitioned, check that partitioning fields are equivalently chosen
+ return checkEquivalentFieldPositionsInKeyFields(
+ produced1.getPartitioningFields(), produced2.getPartitioningFields());
+
+ }
+ else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+ produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
+
+ // both are range partitioned, check that partitioning fields are equivalently chosen
+ return checkEquivalentFieldPositionsInKeyFields(
+ produced1.getPartitioningFields(), produced2.getPartitioningFields());
+
+ }
+ else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
+ produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {
+
+ // both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
+ return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
+ produced2.getPartitioningFields().isExactMatch(this.keys2) &&
+ produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
+ produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());
+
+ }
+ else {
+
+ // no other partitioning valid, incl. ANY_PARTITIONING.
+ // For co-groups we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
+ return false;
+ }
+
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ int numRelevantFields = this.keys1.size();
+
+ Ordering prod1 = produced1.getOrdering();
+ Ordering prod2 = produced2.getOrdering();
+
+ if (prod1 == null || prod2 == null) {
+ throw new CompilerException("The given properties do not meet this operators requirements.");
+ }
+
+ // check that order of fields is equivalent
+ if (!checkEquivalentFieldPositionsInKeyFields(
+ prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
+ return false;
+ }
+
+ // check that order directions are equivalent
+ for (int i = 0; i < numRelevantFields; i++) {
+ if (prod1.getOrder(i) != prod2.getOrder(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections();
+
+ if (inputOrders == null || inputOrders.length < this.keys1.size()) {
+ throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator.");
+ } else if (inputOrders.length > this.keys1.size()) {
+ boolean[] tmp = new boolean[this.keys1.size()];
+ System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
+ inputOrders = tmp;
+ }
+
+ return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ GlobalProperties gp = GlobalProperties.combine(in1, in2);
+ if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
+ gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gp.clearUniqueFieldCombinations();
+ return gp;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ LocalProperties comb = LocalProperties.combine(in1, in2);
+ return comb.clearUniqueFieldSets();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
new file mode 100644
index 0000000..8e7edeb
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class CoGroupWithSolutionSetFirstDescriptor extends CoGroupDescriptor {
+
+ public CoGroupWithSolutionSetFirstDescriptor(FieldList keys1, FieldList keys2) {
+ super(keys1, keys2);
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ RequestedLocalProperties none = new RequestedLocalProperties();
+ RequestedLocalProperties sort = new RequestedLocalProperties(Utils.createOrdering(this.keys2));
+ return Collections.singletonList(new LocalPropertiesPair(none, sort));
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ boolean[] inputOrders = in2.getLocalProperties().getOrdering() == null ? null : in2.getLocalProperties().getOrdering().getFieldSortDirections();
+
+ if (inputOrders == null || inputOrders.length < this.keys2.size()) {
+ throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator.");
+ } else if (inputOrders.length > this.keys2.size()) {
+ boolean[] tmp = new boolean[this.keys2.size()];
+ System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
+ inputOrders = tmp;
+ }
+
+ return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ return true;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return in2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
new file mode 100644
index 0000000..016eb79
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetSecondDescriptor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.util.Utils;
+
+/**
+ *
+ */
+public class CoGroupWithSolutionSetSecondDescriptor extends CoGroupDescriptor {
+
+ public CoGroupWithSolutionSetSecondDescriptor(FieldList keys1, FieldList keys2) {
+ super(keys1, keys2);
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ RequestedLocalProperties sort = new RequestedLocalProperties(Utils.createOrdering(this.keys1));
+ RequestedLocalProperties none = new RequestedLocalProperties();
+ return Collections.singletonList(new LocalPropertiesPair(sort, none));
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ return true;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return in1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
new file mode 100644
index 0000000..bcd4d73
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class CollectorMapDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.COLLECTOR_MAP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.COLLECTOR_MAP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setAnyDistribution();
+ return Collections.singletonList(rgp);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
new file mode 100644
index 0000000..4a467dc
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterFirstDescriptor.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class CrossBlockOuterFirstDescriptor extends CartesianProductDescriptor {
+
+ public CrossBlockOuterFirstDescriptor() {
+ this(true, true);
+ }
+
+ public CrossBlockOuterFirstDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
+ super(allowBroadcastFirst, allowBroadcastSecond);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_FIRST;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return new LocalProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
new file mode 100644
index 0000000..212e048
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossBlockOuterSecondDescriptor.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class CrossBlockOuterSecondDescriptor extends CartesianProductDescriptor {
+
+ public CrossBlockOuterSecondDescriptor() {
+ this(true, true);
+ }
+
+ public CrossBlockOuterSecondDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
+ super(allowBroadcastFirst, allowBroadcastSecond);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return new LocalProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
new file mode 100644
index 0000000..9c7bd63
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterFirstDescriptor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class CrossStreamOuterFirstDescriptor extends CartesianProductDescriptor {
+
+ public CrossStreamOuterFirstDescriptor() {
+ this(true, true);
+ }
+
+ public CrossStreamOuterFirstDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
+ super(allowBroadcastFirst, allowBroadcastSecond);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.NESTEDLOOP_STREAMED_OUTER_FIRST;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ // uniqueness becomes grouping with streamed nested loops
+ if ((in1.getGroupedFields() == null || in1.getGroupedFields().size() == 0) &&
+ in1.getUniqueFields() != null && in1.getUniqueFields().size() > 0)
+ {
+ return LocalProperties.forGrouping(in1.getUniqueFields().iterator().next().toFieldList());
+ } else {
+ return in1.clearUniqueFieldSets();
+ }
+ }
+}
[31/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
deleted file mode 100644
index 04bc527..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
+++ /dev/null
@@ -1,1578 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plantranslate;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.aggregators.AggregatorRegistry;
-import org.apache.flink.api.common.aggregators.AggregatorWithName;
-import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
-import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.operators.util.UserCodeWrapper;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
-import org.apache.flink.runtime.iterative.convergence.WorksetEmptyConvergenceCriterion;
-import org.apache.flink.runtime.iterative.task.IterationHeadPactTask;
-import org.apache.flink.runtime.iterative.task.IterationIntermediatePactTask;
-import org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask;
-import org.apache.flink.runtime.iterative.task.IterationTailPactTask;
-import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
-import org.apache.flink.runtime.jobgraph.DistributionPattern;
-import org.apache.flink.runtime.jobgraph.InputFormatVertex;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.OutputFormatVertex;
-import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
-import org.apache.flink.runtime.operators.CoGroupDriver;
-import org.apache.flink.runtime.operators.CoGroupWithSolutionSetFirstDriver;
-import org.apache.flink.runtime.operators.CoGroupWithSolutionSetSecondDriver;
-import org.apache.flink.runtime.operators.DataSinkTask;
-import org.apache.flink.runtime.operators.DataSourceTask;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.JoinWithSolutionSetFirstDriver;
-import org.apache.flink.runtime.operators.JoinWithSolutionSetSecondDriver;
-import org.apache.flink.runtime.operators.MatchDriver;
-import org.apache.flink.runtime.operators.NoOpDriver;
-import org.apache.flink.runtime.operators.RegularPactTask;
-import org.apache.flink.runtime.operators.chaining.ChainedDriver;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.Visitor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * This component translates the optimizer's resulting {@link org.apache.flink.optimizer.plan.OptimizedPlan}
- * to a {@link org.apache.flink.runtime.jobgraph.JobGraph}. The translation is not strictly a one-to-one,
- * because some nodes from the OptimizedPlan are collapsed into one job vertex.
- *
- * This translation does not make any decisions or assumptions. All degrees-of-freedom in the execution
- * of the job are made by the Optimizer, so that this translation becomes a deterministic mapping.
- *
- * The basic method of operation is a top down traversal over the plan graph. On the way down, job vertices
- * are created for the plan nodes, on the way back up, the nodes connect their predecessors.
- */
-public class JobGraphGenerator implements Visitor<PlanNode> {
-
- public static final String MERGE_ITERATION_AUX_TASKS_KEY = "compiler.merge-iteration-aux";
-
- private static final boolean mergeIterationAuxTasks = GlobalConfiguration.getBoolean(MERGE_ITERATION_AUX_TASKS_KEY, false);
-
- private static final TaskInChain ALREADY_VISITED_PLACEHOLDER = new TaskInChain(null, null, null);
-
- // ------------------------------------------------------------------------
-
- private Map<PlanNode, AbstractJobVertex> vertices; // a map from optimizer nodes to job vertices
-
- private Map<PlanNode, TaskInChain> chainedTasks; // a map from optimizer nodes to job vertices
-
- private Map<IterationPlanNode, IterationDescriptor> iterations;
-
- private List<TaskInChain> chainedTasksInSequence;
-
- private List<AbstractJobVertex> auxVertices; // auxiliary vertices which are added during job graph generation
-
- private final int defaultMaxFan;
-
- private final float defaultSortSpillingThreshold;
-
- private int iterationIdEnumerator = 1;
-
- private IterationPlanNode currentIteration; // the current the enclosing iteration
-
- private List<IterationPlanNode> iterationStack; // stack of enclosing iterations
-
- private SlotSharingGroup sharingGroup;
-
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new job graph generator that uses the default values for its resource configuration.
- */
- public JobGraphGenerator() {
- this.defaultMaxFan = ConfigConstants.DEFAULT_SPILLING_MAX_FAN;
- this.defaultSortSpillingThreshold = ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD;
- }
-
- public JobGraphGenerator(Configuration config) {
- this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY,
- ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
- this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
- ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
- }
-
- /**
- * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a
- * {@link org.apache.flink.runtime.jobgraph.JobGraph}.
- *
- * @param program Optimized plan that is translated into a JobGraph.
- * @return JobGraph generated frmo the plan.
- */
- public JobGraph compileJobGraph(OptimizedPlan program) {
- this.vertices = new HashMap<PlanNode, AbstractJobVertex>();
- this.chainedTasks = new HashMap<PlanNode, TaskInChain>();
- this.chainedTasksInSequence = new ArrayList<TaskInChain>();
- this.auxVertices = new ArrayList<AbstractJobVertex>();
- this.iterations = new HashMap<IterationPlanNode, IterationDescriptor>();
- this.iterationStack = new ArrayList<IterationPlanNode>();
-
- this.sharingGroup = new SlotSharingGroup();
-
- // this starts the traversal that generates the job graph
- program.accept(this);
-
- // sanity check that we are not somehow in an iteration at the end
- if (this.currentIteration != null) {
- throw new CompilerException("The graph translation ended prematurely, leaving an unclosed iteration.");
- }
-
- // finalize the iterations
- for (IterationDescriptor iteration : this.iterations.values()) {
- if (iteration.getIterationNode() instanceof BulkIterationPlanNode) {
- finalizeBulkIteration(iteration);
- } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) {
- finalizeWorksetIteration(iteration);
- } else {
- throw new CompilerException();
- }
- }
-
- // now that the traversal is done, we have the chained tasks write their configs into their
- // parents' configurations
- for (TaskInChain tic : this.chainedTasksInSequence) {
- TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
- t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
- }
-
- // create the job graph object
- JobGraph graph = new JobGraph(program.getJobName());
- graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());
- graph.setAllowQueuedScheduling(false);
-
- // add vertices to the graph
- for (AbstractJobVertex vertex : this.vertices.values()) {
- graph.addVertex(vertex);
- }
-
- for (AbstractJobVertex vertex : this.auxVertices) {
- graph.addVertex(vertex);
- vertex.setSlotSharingGroup(sharingGroup);
- }
-
- // add registered cache file into job configuration
- for (Entry<String, DistributedCacheEntry> e : program.getOriginalPactPlan().getCachedFiles()) {
- DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
- }
-
- try {
- InstantiationUtil.writeObjectToConfig(
- program.getOriginalPactPlan().getExecutionConfig(),
- graph.getJobConfiguration(),
- ExecutionConfig.CONFIG_KEY);
- } catch (IOException e) {
- throw new RuntimeException("Config object could not be written to Job Configuration: " + e);
- }
-
- // release all references again
- this.vertices = null;
- this.chainedTasks = null;
- this.chainedTasksInSequence = null;
- this.auxVertices = null;
- this.iterations = null;
- this.iterationStack = null;
-
- // return job graph
- return graph;
- }
-
- /**
- * This methods implements the pre-visiting during a depth-first traversal. It create the job vertex and
- * sets local strategy.
- *
- * @param node
- * The node that is currently processed.
- * @return True, if the visitor should descend to the node's children, false if not.
- * @see org.apache.flink.util.Visitor#preVisit(org.apache.flink.util.Visitable)
- */
- @Override
- public boolean preVisit(PlanNode node) {
- // check if we have visited this node before. in non-tree graphs, this happens
- if (this.vertices.containsKey(node) || this.chainedTasks.containsKey(node) || this.iterations.containsKey(node)) {
- // return false to prevent further descend
- return false;
- }
-
- // the vertex to be created for the current node
- final AbstractJobVertex vertex;
- try {
- if (node instanceof SinkPlanNode) {
- vertex = createDataSinkVertex((SinkPlanNode) node);
- }
- else if (node instanceof SourcePlanNode) {
- vertex = createDataSourceVertex((SourcePlanNode) node);
- }
- else if (node instanceof BulkIterationPlanNode) {
- BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
- // for the bulk iteration, we skip creating anything for now. we create the graph
- // for the step function in the post visit.
-
- // check that the root of the step function has the same DOP as the iteration.
- // because the tail must have the same DOP as the head, we can only merge the last
- // operator with the tail, if they have the same DOP. not merging is currently not
- // implemented
- PlanNode root = iterationNode.getRootOfStepFunction();
- if (root.getParallelism() != node.getParallelism())
- {
- throw new CompilerException("Error: The final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
-
- IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
- this.iterations.put(iterationNode, descr);
- vertex = null;
- }
- else if (node instanceof WorksetIterationPlanNode) {
- WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
-
- // we have the same constraints as for the bulk iteration
- PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
- PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
-
- if (nextWorkSet.getParallelism() != node.getParallelism())
- {
- throw new CompilerException("It is currently not supported that the final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
- if (solutionSetDelta.getParallelism() != node.getParallelism())
- {
- throw new CompilerException("It is currently not supported that the final operator of the step " +
- "function has a different degree of parallelism than the iteration operator itself.");
- }
-
- IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
- this.iterations.put(iterationNode, descr);
- vertex = null;
- }
- else if (node instanceof SingleInputPlanNode) {
- vertex = createSingleInputVertex((SingleInputPlanNode) node);
- }
- else if (node instanceof DualInputPlanNode) {
- vertex = createDualInputVertex((DualInputPlanNode) node);
- }
- else if (node instanceof NAryUnionPlanNode) {
- // skip the union for now
- vertex = null;
- }
- else if (node instanceof BulkPartialSolutionPlanNode) {
- // create a head node (or not, if it is merged into its successor)
- vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
- }
- else if (node instanceof SolutionSetPlanNode) {
- // this represents an access into the solution set index.
- // we do not create a vertex for the solution set here (we create the head at the workset place holder)
-
- // we adjust the joins / cogroups that go into the solution set here
- for (Channel c : node.getOutgoingChannels()) {
- DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
- AbstractJobVertex accessingVertex = this.vertices.get(target);
- TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
- int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
-
- // sanity checks
- if (inputNum == -1) {
- throw new CompilerException();
- }
-
- // adjust the driver
- if (conf.getDriver().equals(MatchDriver.class)) {
- conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
- }
- else if (conf.getDriver().equals(CoGroupDriver.class)) {
- conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
- }
- else {
- throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
- }
- }
-
- // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
- this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
-
- vertex = null;
- }
- else if (node instanceof WorksetPlanNode) {
- // create the iteration head here
- vertex = createWorksetIterationHead((WorksetPlanNode) node);
- }
- else {
- throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
- }
- }
- catch (Exception e) {
- throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
- }
-
- // check if a vertex was created, or if it was chained or skipped
- if (vertex != null) {
- // set degree of parallelism
- int pd = node.getParallelism();
- vertex.setParallelism(pd);
-
- vertex.setSlotSharingGroup(sharingGroup);
-
- // check whether this vertex is part of an iteration step function
- if (this.currentIteration != null) {
- // check that the task has the same DOP as the iteration as such
- PlanNode iterationNode = (PlanNode) this.currentIteration;
- if (iterationNode.getParallelism() < pd) {
- throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
- }
-
- // store the id of the iterations the step functions participate in
- IterationDescriptor descr = this.iterations.get(this.currentIteration);
- new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
- }
-
- // store in the map
- this.vertices.put(node, vertex);
- }
-
- // returning true causes deeper descend
- return true;
- }
-
- /**
- * This method implements the post-visit during the depth-first traversal. When the post visit happens,
- * all of the descendants have been processed, so this method connects all of the current node's
- * predecessors to the current node.
- *
- * @param node
- * The node currently processed during the post-visit.
- * @see org.apache.flink.util.Visitor#postVisit(org.apache.flink.util.Visitable) t
- */
- @Override
- public void postVisit(PlanNode node) {
- try {
- // --------- check special cases for which we handle post visit differently ----------
-
- // skip data source node (they have no inputs)
- // also, do nothing for union nodes, we connect them later when gathering the inputs for a task
- // solution sets have no input. the initial solution set input is connected when the iteration node is in its postVisit
- if (node instanceof SourcePlanNode || node instanceof NAryUnionPlanNode || node instanceof SolutionSetPlanNode) {
- return;
- }
-
- // check if we have an iteration. in that case, translate the step function now
- if (node instanceof IterationPlanNode) {
- // prevent nested iterations
- if (node.isOnDynamicPath()) {
- throw new CompilerException("Nested Iterations are not possible at the moment!");
- }
-
- // if we recursively go into an iteration (because the constant path of one iteration contains
- // another one), we push the current one onto the stack
- if (this.currentIteration != null) {
- this.iterationStack.add(this.currentIteration);
- }
-
- this.currentIteration = (IterationPlanNode) node;
- this.currentIteration.acceptForStepFunction(this);
-
- // pop the current iteration from the stack
- if (this.iterationStack.isEmpty()) {
- this.currentIteration = null;
- } else {
- this.currentIteration = this.iterationStack.remove(this.iterationStack.size() - 1);
- }
-
- // inputs for initial bulk partial solution or initial workset are already connected to the iteration head in the head's post visit.
- // connect the initial solution set now.
- if (node instanceof WorksetIterationPlanNode) {
- // connect the initial solution set
- WorksetIterationPlanNode wsNode = (WorksetIterationPlanNode) node;
- AbstractJobVertex headVertex = this.iterations.get(wsNode).getHeadTask();
- TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- int inputIndex = headConfig.getDriverStrategy().getNumInputs();
- headConfig.setIterationHeadSolutionSetInputIndex(inputIndex);
- translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
- }
-
- return;
- }
-
- final AbstractJobVertex targetVertex = this.vertices.get(node);
-
-
- // --------- Main Path: Translation of channels ----------
- //
- // There are two paths of translation: One for chained tasks (or merged tasks in general),
- // which do not have their own task vertex. The other for tasks that have their own vertex,
- // or are the primary task in a vertex (to which the others are chained).
-
- // check whether this node has its own task, or is merged with another one
- if (targetVertex == null) {
- // node's task is merged with another task. it is either chained, of a merged head vertex
- // from an iteration
- final TaskInChain chainedTask;
- if ((chainedTask = this.chainedTasks.get(node)) != null) {
- // Chained Task. Sanity check first...
- final Iterator<Channel> inConns = node.getInputs().iterator();
- if (!inConns.hasNext()) {
- throw new CompilerException("Bug: Found chained task with no input.");
- }
- final Channel inConn = inConns.next();
-
- if (inConns.hasNext()) {
- throw new CompilerException("Bug: Found a chained task with more than one input!");
- }
- if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
- throw new CompilerException("Bug: Found a chained task with an input local strategy.");
- }
- if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
- throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
- }
-
- AbstractJobVertex container = chainedTask.getContainingVertex();
-
- if (container == null) {
- final PlanNode sourceNode = inConn.getSource();
- container = this.vertices.get(sourceNode);
- if (container == null) {
- // predecessor is itself chained
- container = this.chainedTasks.get(sourceNode).getContainingVertex();
- if (container == null) {
- throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
- }
- } else {
- // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
- new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
- }
- chainedTask.setContainingVertex(container);
- }
-
- // add info about the input serializer type
- chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
-
- // update name of container task
- String containerTaskName = container.getName();
- if(containerTaskName.startsWith("CHAIN ")) {
- container.setName(containerTaskName+" -> "+chainedTask.getTaskName());
- } else {
- container.setName("CHAIN "+containerTaskName+" -> "+chainedTask.getTaskName());
- }
-
- this.chainedTasksInSequence.add(chainedTask);
- return;
- }
- else if (node instanceof BulkPartialSolutionPlanNode ||
- node instanceof WorksetPlanNode)
- {
- // merged iteration head task. the task that the head is merged with will take care of it
- return;
- } else {
- throw new CompilerException("Bug: Unrecognized merged task vertex.");
- }
- }
-
- // -------- Here, we translate non-chained tasks -------------
-
-
- if (this.currentIteration != null) {
- AbstractJobVertex head = this.iterations.get(this.currentIteration).getHeadTask();
- // the head may still be null if we descend into the static parts first
- if (head != null) {
- targetVertex.setStrictlyCoLocatedWith(head);
- }
- }
-
-
- // create the config that will contain all the description of the inputs
- final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
-
- // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
- // enclosing iteration node, because the inputs are the initial inputs to the iteration.
- final Iterator<Channel> inConns;
- if (node instanceof BulkPartialSolutionPlanNode) {
- inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
- // because the partial solution has its own vertex, is has only one (logical) input.
- // note this in the task configuration
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
- } else if (node instanceof WorksetPlanNode) {
- WorksetPlanNode wspn = (WorksetPlanNode) node;
- // input that is the initial workset
- inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
-
- // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
- // be input 0 and the solution set will be input 1
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
- targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
- } else {
- inConns = node.getInputs().iterator();
- }
- if (!inConns.hasNext()) {
- throw new CompilerException("Bug: Found a non-source task with no input.");
- }
-
- int inputIndex = 0;
- while (inConns.hasNext()) {
- Channel input = inConns.next();
- inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
- }
- // broadcast variables
- int broadcastInputIndex = 0;
- for (NamedChannel broadcastInput: node.getBroadcastInputs()) {
- int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true);
- targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
- targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex);
- broadcastInputIndex += broadcastInputIndexDelta;
- }
- } catch (Exception e) {
- throw new CompilerException(
- "An error occurred while translating the optimized plan to a nephele JobGraph: " + e.getMessage(), e);
- }
- }
-
- private int translateChannel(Channel input, int inputIndex, AbstractJobVertex targetVertex,
- TaskConfig targetVertexConfig, boolean isBroadcast) throws Exception
- {
- final PlanNode inputPlanNode = input.getSource();
- final Iterator<Channel> allInChannels;
-
- if (inputPlanNode instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) inputPlanNode).getListOfInputs().iterator();
- }
- else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
- if (this.vertices.get(inputPlanNode) == null) {
- // merged iteration head
- final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
- final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
-
- // check if the iteration's input is a union
- if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs().iterator();
- } else {
- allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
- }
-
- // also, set the index of the gate with the partial solution
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
- } else {
- // standalone iteration head
- allInChannels = Collections.singletonList(input).iterator();
- }
- } else if (inputPlanNode instanceof WorksetPlanNode) {
- if (this.vertices.get(inputPlanNode) == null) {
- // merged iteration head
- final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
- final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
-
- // check if the iteration's input is a union
- if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
- allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs().iterator();
- } else {
- allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
- }
-
- // also, set the index of the gate with the partial solution
- targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
- } else {
- // standalone iteration head
- allInChannels = Collections.singletonList(input).iterator();
- }
- } else if (inputPlanNode instanceof SolutionSetPlanNode) {
- // for now, skip connections with the solution set node, as this is a local index access (later to be parameterized here)
- // rather than a vertex connection
- return 0;
- } else {
- allInChannels = Collections.singletonList(input).iterator();
- }
-
- // check that the type serializer is consistent
- TypeSerializerFactory<?> typeSerFact = null;
-
- // accounting for channels on the dynamic path
- int numChannelsTotal = 0;
- int numChannelsDynamicPath = 0;
- int numDynamicSenderTasksTotal = 0;
-
-
- // expand the channel to all the union channels, in case there is a union operator at its source
- while (allInChannels.hasNext()) {
- final Channel inConn = allInChannels.next();
-
- // sanity check the common serializer
- if (typeSerFact == null) {
- typeSerFact = inConn.getSerializer();
- } else if (!typeSerFact.equals(inConn.getSerializer())) {
- throw new CompilerException("Conflicting types in union operator.");
- }
-
- final PlanNode sourceNode = inConn.getSource();
- AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
- TaskConfig sourceVertexConfig;
-
- if (sourceVertex == null) {
- // this predecessor is chained to another task or an iteration
- final TaskInChain chainedTask;
- final IterationDescriptor iteration;
- if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
- // push chained task
- if (chainedTask.getContainingVertex() == null) {
- throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
- }
- sourceVertex = chainedTask.getContainingVertex();
- sourceVertexConfig = chainedTask.getTaskConfig();
- } else if ((iteration = this.iterations.get(sourceNode)) != null) {
- // predecessor is an iteration
- sourceVertex = iteration.getHeadTask();
- sourceVertexConfig = iteration.getHeadFinalResultConfig();
- } else {
- throw new CompilerException("Bug: Could not resolve source node for a channel.");
- }
- } else {
- // predecessor is its own vertex
- sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
- }
- DistributionPattern pattern = connectJobVertices(
- inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
-
- // accounting on channels and senders
- numChannelsTotal++;
- if (inConn.isOnDynamicPath()) {
- numChannelsDynamicPath++;
- numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
- sourceVertex.getParallelism(), targetVertex.getParallelism());
- }
- }
-
- // for the iterations, check that the number of dynamic channels is the same as the number
- // of channels for this logical input. this condition is violated at the moment, if there
- // is a union between nodes on the static and nodes on the dynamic path
- if (numChannelsDynamicPath > 0 && numChannelsTotal != numChannelsDynamicPath) {
- throw new CompilerException("Error: It is currently not supported to union between dynamic and static path in an iteration.");
- }
- if (numDynamicSenderTasksTotal > 0) {
- if (isBroadcast) {
- targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
- } else {
- targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
- }
- }
-
- // the local strategy is added only once. in non-union case that is the actual edge,
- // in the union case, it is the edge between union and the target node
- addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
- return 1;
- }
-
- private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) {
- if (pattern == DistributionPattern.ALL_TO_ALL) {
- return numSenders;
- } else if (pattern == DistributionPattern.POINTWISE) {
- if (numSenders != numReceivers) {
- if (numReceivers == 1) {
- return numSenders;
- }
- else if (numSenders == 1) {
- return 1;
- }
- else {
- throw new CompilerException("Error: A changing degree of parallelism is currently " +
- "not supported between tasks within an iteration.");
- }
- } else {
- return 1;
- }
- } else {
- throw new CompilerException("Unknown distribution pattern for channels: " + pattern);
- }
- }
-
- // ------------------------------------------------------------------------
- // Methods for creating individual vertices
- // ------------------------------------------------------------------------
-
- private AbstractJobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
- final String taskName = node.getNodeName();
- final DriverStrategy ds = node.getDriverStrategy();
-
- // check, whether chaining is possible
- boolean chaining = false;
- {
- Channel inConn = node.getInput();
- PlanNode pred = inConn.getSource();
- chaining = ds.getPushChainDriverClass() != null &&
- !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged
- !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways
- !(pred instanceof WorksetPlanNode) && // workset merges anyways
- !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
- inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
- inConn.getLocalStrategy() == LocalStrategy.NONE &&
- pred.getOutgoingChannels().size() == 1 &&
- node.getParallelism() == pred.getParallelism() &&
- node.getBroadcastInputs().isEmpty();
-
- // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
- // in a tail
- if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
- node.getOutgoingChannels().size() > 0)
- {
- WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
- if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
- chaining = false;
- }
- }
- // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
- if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
- {
- BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
- if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
- chaining = false;
- }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
- wspn.getRootOfTerminationCriterion() == pred)) {
- chaining = false;
- }
- }
- }
-
- final AbstractJobVertex vertex;
- final TaskConfig config;
-
- if (chaining) {
- vertex = null;
- config = new TaskConfig(new Configuration());
- this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
- } else {
- // create task vertex
- vertex = new AbstractJobVertex(taskName);
- vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
-
- config = new TaskConfig(vertex.getConfiguration());
- config.setDriver(ds.getDriverClass());
- }
-
- // set user code
- config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
- config.setStubParameters(node.getProgramOperator().getParameters());
-
- // set the driver strategy
- config.setDriverStrategy(ds);
- for(int i=0;i<ds.getNumRequiredComparators();i++) {
- config.setDriverComparator(node.getComparator(i), i);
- }
- // assign memory, file-handles, etc.
- assignDriverResources(node, config);
- return vertex;
- }
-
- private AbstractJobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
- final String taskName = node.getNodeName();
- final DriverStrategy ds = node.getDriverStrategy();
- final AbstractJobVertex vertex = new AbstractJobVertex(taskName);
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
- vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
-
- // set user code
- config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
- config.setStubParameters(node.getProgramOperator().getParameters());
-
- // set the driver strategy
- config.setDriver(ds.getDriverClass());
- config.setDriverStrategy(ds);
- if (node.getComparator1() != null) {
- config.setDriverComparator(node.getComparator1(), 0);
- }
- if (node.getComparator2() != null) {
- config.setDriverComparator(node.getComparator2(), 1);
- }
- if (node.getPairComparator() != null) {
- config.setDriverPairComparator(node.getPairComparator());
- }
-
- // assign memory, file-handles, etc.
- assignDriverResources(node, config);
- return vertex;
- }
-
- private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
- final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
-
- vertex.setInvokableClass(DataSourceTask.class);
- vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
-
- // set user code
- config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
- config.setStubParameters(node.getProgramOperator().getParameters());
-
- config.setOutputSerializer(node.getSerializer());
- return vertex;
- }
-
- private AbstractJobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
- final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName());
- final TaskConfig config = new TaskConfig(vertex.getConfiguration());
-
- vertex.setInvokableClass(DataSinkTask.class);
- vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
-
- // set user code
- config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
- config.setStubParameters(node.getProgramOperator().getParameters());
-
- return vertex;
- }
-
- private AbstractJobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
- // get the bulk iteration that corresponds to this partial solution node
- final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
-
- // check whether we need an individual vertex for the partial solution, or whether we
- // attach ourselves to the vertex of the parent node. We can combine the head with a node of
- // the step function, if
- // 1) There is one parent that the partial solution connects to via a forward pattern and no
- // local strategy
- // 2) DOP and the number of subtasks per instance does not change
- // 3) That successor is not a union
- // 4) That successor is not itself the last node of the step function
- // 5) There is no local strategy on the edge for the initial partial solution, as
- // this translates to a local strategy that would only be executed in the first iteration
-
- final boolean merge;
- if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
- final Channel c = pspn.getOutgoingChannels().get(0);
- final PlanNode successor = c.getTarget();
- merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
- c.getLocalStrategy() == LocalStrategy.NONE &&
- c.getTempMode() == TempMode.NONE &&
- successor.getParallelism() == pspn.getParallelism() &&
- !(successor instanceof NAryUnionPlanNode) &&
- successor != iteration.getRootOfStepFunction() &&
- iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
- } else {
- merge = false;
- }
-
- // create or adopt the head vertex
- final AbstractJobVertex toReturn;
- final AbstractJobVertex headVertex;
- final TaskConfig headConfig;
- if (merge) {
- final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
- headVertex = (AbstractJobVertex) this.vertices.get(successor);
-
- if (headVertex == null) {
- throw new CompilerException(
- "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
- }
-
- // reset the vertex type to iteration head
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- toReturn = null;
- } else {
- // instantiate the head vertex and give it a no-op driver as the driver strategy.
- // everything else happens in the post visit, after the input (the initial partial solution)
- // is connected.
- headVertex = new AbstractJobVertex("PartialSolution ("+iteration.getNodeName()+")");
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- headConfig.setDriver(NoOpDriver.class);
- toReturn = headVertex;
- }
-
- // create the iteration descriptor and the iteration to it
- IterationDescriptor descr = this.iterations.get(iteration);
- if (descr == null) {
- throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
- }
- descr.setHeadTask(headVertex, headConfig);
-
- return toReturn;
- }
-
- private AbstractJobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
- // get the bulk iteration that corresponds to this partial solution node
- final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
-
- // check whether we need an individual vertex for the partial solution, or whether we
- // attach ourselves to the vertex of the parent node. We can combine the head with a node of
- // the step function, if
- // 1) There is one parent that the partial solution connects to via a forward pattern and no
- // local strategy
- // 2) DOP and the number of subtasks per instance does not change
- // 3) That successor is not a union
- // 4) That successor is not itself the last node of the step function
- // 5) There is no local strategy on the edge for the initial workset, as
- // this translates to a local strategy that would only be executed in the first superstep
-
- final boolean merge;
- if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
- final Channel c = wspn.getOutgoingChannels().get(0);
- final PlanNode successor = c.getTarget();
- merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
- c.getLocalStrategy() == LocalStrategy.NONE &&
- c.getTempMode() == TempMode.NONE &&
- successor.getParallelism() == wspn.getParallelism() &&
- !(successor instanceof NAryUnionPlanNode) &&
- successor != iteration.getNextWorkSetPlanNode() &&
- iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
- } else {
- merge = false;
- }
-
- // create or adopt the head vertex
- final AbstractJobVertex toReturn;
- final AbstractJobVertex headVertex;
- final TaskConfig headConfig;
- if (merge) {
- final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
- headVertex = (AbstractJobVertex) this.vertices.get(successor);
-
- if (headVertex == null) {
- throw new CompilerException(
- "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
- }
-
- // reset the vertex type to iteration head
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- toReturn = null;
- } else {
- // instantiate the head vertex and give it a no-op driver as the driver strategy.
- // everything else happens in the post visit, after the input (the initial partial solution)
- // is connected.
- headVertex = new AbstractJobVertex("IterationHead("+iteration.getNodeName()+")");
- headVertex.setInvokableClass(IterationHeadPactTask.class);
- headConfig = new TaskConfig(headVertex.getConfiguration());
- headConfig.setDriver(NoOpDriver.class);
- toReturn = headVertex;
- }
-
- headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
-
- // create the iteration descriptor and the iteration to it
- IterationDescriptor descr = this.iterations.get(iteration);
- if (descr == null) {
- throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
- }
- descr.setHeadTask(headVertex, headConfig);
-
- return toReturn;
- }
-
- private void assignDriverResources(PlanNode node, TaskConfig config) {
- final double relativeMem = node.getRelativeMemoryPerSubTask();
- if (relativeMem > 0) {
- config.setRelativeMemoryDriver(relativeMem);
- config.setFilehandlesDriver(this.defaultMaxFan);
- config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
- }
- }
-
- private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) {
- if (c.getRelativeMemoryLocalStrategy() > 0) {
- config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy());
- config.setFilehandlesInput(inputNum, this.defaultMaxFan);
- config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold);
- }
- }
-
- // ------------------------------------------------------------------------
- // Connecting Vertices
- // ------------------------------------------------------------------------
-
- /**
- * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
- * channel is then the channel into the union node, the local strategy channel the one from the union to the
- * actual target operator.
- *
- * @param channel
- * @param inputNumber
- * @param sourceVertex
- * @param sourceConfig
- * @param targetVertex
- * @param targetConfig
- * @param isBroadcast
- * @throws CompilerException
- */
- private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
- final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
- final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
- throws CompilerException
- {
- // ------------ connect the vertices to the job graph --------------
- final DistributionPattern distributionPattern;
-
- switch (channel.getShipStrategy()) {
- case FORWARD:
- distributionPattern = DistributionPattern.POINTWISE;
- break;
- case PARTITION_RANDOM:
- case BROADCAST:
- case PARTITION_HASH:
- case PARTITION_CUSTOM:
- case PARTITION_RANGE:
- case PARTITION_FORCED_REBALANCE:
- distributionPattern = DistributionPattern.ALL_TO_ALL;
- break;
- default:
- throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
- }
-
- final ResultPartitionType resultType;
-
- switch (channel.getDataExchangeMode()) {
-
- case PIPELINED:
- resultType = ResultPartitionType.PIPELINED;
- break;
-
- case BATCH:
- // BLOCKING results are currently not supported in closed loop iterations
- //
- // See https://issues.apache.org/jira/browse/FLINK-1713 for details
- resultType = channel.getSource().isOnDynamicPath()
- ? ResultPartitionType.PIPELINED
- : ResultPartitionType.BLOCKING;
- break;
-
- case PIPELINE_WITH_BATCH_FALLBACK:
- throw new UnsupportedOperationException("Data exchange mode " +
- channel.getDataExchangeMode() + " currently not supported.");
-
- default:
- throw new UnsupportedOperationException("Unknown data exchange mode.");
-
- }
-
- targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
-
- // -------------- configure the source task's ship strategy strategies in task config --------------
- final int outputIndex = sourceConfig.getNumOutputs();
- sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
- if (outputIndex == 0) {
- sourceConfig.setOutputSerializer(channel.getSerializer());
- }
- if (channel.getShipStrategyComparator() != null) {
- sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
- }
-
- if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
-
- final DataDistribution dataDistribution = channel.getDataDistribution();
- if (dataDistribution != null) {
- sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
- } else {
- throw new RuntimeException("Range partitioning requires data distribution");
- // TODO: inject code and configuration for automatic histogram generation
- }
- }
-
- if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
- if (channel.getPartitioner() != null) {
- sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
- } else {
- throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
- }
- }
-
- // ---------------- configure the receiver -------------------
- if (isBroadcast) {
- targetConfig.addBroadcastInputToGroup(inputNumber);
- } else {
- targetConfig.addInputToGroup(inputNumber);
- }
- return distributionPattern;
- }
-
- private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
- // serializer
- if (isBroadcastChannel) {
- config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
-
- if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
- throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
- } else {
- return;
- }
- } else {
- config.setInputSerializer(channel.getSerializer(), inputNum);
- }
-
- // local strategy
- if (channel.getLocalStrategy() != LocalStrategy.NONE) {
- config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
- if (channel.getLocalStrategyComparator() != null) {
- config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
- }
- }
-
- assignLocalStrategyResources(channel, config, inputNum);
-
- // materialization / caching
- if (channel.getTempMode() != null) {
- final TempMode tm = channel.getTempMode();
-
- boolean needsMemory = false;
- // Don't add a pipeline breaker if the data exchange is already blocking.
- if (tm.breaksPipeline() && channel.getDataExchangeMode() != DataExchangeMode.BATCH) {
- config.setInputAsynchronouslyMaterialized(inputNum, true);
- needsMemory = true;
- }
- if (tm.isCached()) {
- config.setInputCached(inputNum, true);
- needsMemory = true;
- }
-
- if (needsMemory) {
- // sanity check
- if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
- throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
- }
- config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
- }
- }
- }
-
- private void finalizeBulkIteration(IterationDescriptor descr) {
-
- final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
- final AbstractJobVertex headVertex = descr.getHeadTask();
- final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
-
- // ------------ finalize the head config with the final outputs and the sync gate ------------
- final int numStepFunctionOuts = headConfig.getNumOutputs();
- final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
-
- if (numStepFunctionOuts == 0) {
- throw new CompilerException("The iteration has no operation inside the step function.");
- }
-
- headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
- headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
- final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
- if (relativeMemForBackChannel <= 0) {
- throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
- }
- headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
-
- // --------------------------- create the sync task ---------------------------
- final AbstractJobVertex sync = new AbstractJobVertex("Sync(" + bulkNode.getNodeName() + ")");
- sync.setInvokableClass(IterationSynchronizationSinkTask.class);
- sync.setParallelism(1);
- this.auxVertices.add(sync);
-
- final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
- syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
-
- // set the number of iteration / convergence criterion for the sync
- final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
- if (maxNumIterations < 1) {
- throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
- }
- syncConfig.setNumberOfIterations(maxNumIterations);
-
- // connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
-
- // ----------------------------- create the iteration tail ------------------------------
-
- final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
- final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
- final TaskConfig tailConfig;
-
- AbstractJobVertex rootOfStepFunctionVertex = (AbstractJobVertex) this.vertices.get(rootOfStepFunction);
- if (rootOfStepFunctionVertex == null) {
- // last op is chained
- final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
- }
- rootOfStepFunctionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
-
- // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
- tailConfig = taskInChain.getTaskConfig();
- } else {
- tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
- }
-
- tailConfig.setIsWorksetUpdate();
-
- // No following termination criterion
- if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
-
- rootOfStepFunctionVertex.setInvokableClass(IterationTailPactTask.class);
-
- tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
- }
-
-
- // create the fake output task for termination criterion, if needed
- final TaskConfig tailConfigOfTerminationCriterion;
- // If we have a termination criterion and it is not an intermediate node
- if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
- AbstractJobVertex rootOfTerminationCriterionVertex = (AbstractJobVertex) this.vertices.get(rootOfTerminationCriterion);
-
-
- if (rootOfTerminationCriterionVertex == null) {
- // last op is chained
- final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
- }
- rootOfTerminationCriterionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
-
- // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
- tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
- } else {
- tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
- }
-
- rootOfTerminationCriterionVertex.setInvokableClass(IterationTailPactTask.class);
- // Hack
- tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
- tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
-
- // tell the head that it needs to wait for the solution set updates
- headConfig.setWaitForSolutionSetUpdate();
- }
-
- // ------------------- register the aggregators -------------------
- AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
- Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
-
- headConfig.addIterationAggregators(allAggregators);
- syncConfig.addIterationAggregators(allAggregators);
-
- String convAggName = aggs.getConvergenceCriterionAggregatorName();
- ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
-
- if (convCriterion != null || convAggName != null) {
- if (convCriterion == null) {
- throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
- }
- if (convAggName == null) {
- throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
- }
-
- syncConfig.setConvergenceCriterion(convAggName, convCriterion);
- }
- }
-
- private void finalizeWorksetIteration(IterationDescriptor descr) {
- final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
- final AbstractJobVertex headVertex = descr.getHeadTask();
- final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
- final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
-
- // ------------ finalize the head config with the final outputs and the sync gate ------------
- {
- final int numStepFunctionOuts = headConfig.getNumOutputs();
- final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
-
- if (numStepFunctionOuts == 0) {
- throw new CompilerException("The workset iteration has no operation on the workset inside the step function.");
- }
-
- headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
- headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
- final double relativeMemory = iterNode.getRelativeMemoryPerSubTask();
- if (relativeMemory <= 0) {
- throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
- }
-
- headConfig.setIsWorksetIteration();
- headConfig.setRelativeBackChannelMemory(relativeMemory / 2);
- headConfig.setRelativeSolutionSetMemory(relativeMemory / 2);
-
- // set the solution set serializer and comparator
- headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
- headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
- }
-
- // --------------------------- create the sync task ---------------------------
- final TaskConfig syncConfig;
- {
- final AbstractJobVertex sync = new AbstractJobVertex("Sync (" + iterNode.getNodeName() + ")");
- sync.setInvokableClass(IterationSynchronizationSinkTask.class);
- sync.setParallelism(1);
- this.auxVertices.add(sync);
-
- syncConfig = new TaskConfig(sync.getConfiguration());
- syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
-
- // set the number of iteration / convergence criterion for the sync
- final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
- if (maxNumIterations < 1) {
- throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
- }
- syncConfig.setNumberOfIterations(maxNumIterations);
-
- // connect the sync task
- sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
- }
-
- // ----------------------------- create the iteration tails -----------------------------
- // ----------------------- for next workset and solution set delta-----------------------
-
- {
- // we have three possible cases:
- // 1) Two tails, one for workset update, one for solution set update
- // 2) One tail for workset update, solution set update happens in an intermediate task
- // 3) One tail for solution set update, workset update happens in an intermediate task
-
- final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
- final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
-
- final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
- final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
-
- {
- // get the vertex for the workset update
- final TaskConfig worksetTailConfig;
- AbstractJobVertex nextWorksetVertex = (AbstractJobVertex) this.vertices.get(nextWorksetNode);
- if (nextWorksetVertex == null) {
- // nextWorksetVertex is chained
- TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
- }
- nextWorksetVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
- worksetTailConfig = taskInChain.getTaskConfig();
- } else {
- worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
- }
-
- // mark the node to perform workset updates
- worksetTailConfig.setIsWorksetIteration();
- worksetTailConfig.setIsWorksetUpdate();
-
- if (hasWorksetTail) {
- nextWorksetVertex.setInvokableClass(IterationTailPactTask.class);
-
- worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
- }
- }
- {
- final TaskConfig solutionDeltaConfig;
- AbstractJobVertex solutionDeltaVertex = (AbstractJobVertex) this.vertices.get(solutionDeltaNode);
- if (solutionDeltaVertex == null) {
- // last op is chained
- TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
- if (taskInChain == null) {
- throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
- }
- solutionDeltaVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
- solutionDeltaConfig = taskInChain.getTaskConfig();
- } else {
- solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
- }
-
- solutionDeltaConfig.setIsWorksetIteration();
- solutionDeltaConfig.setIsSolutionSetUpdate();
-
- if (hasSolutionSetTail) {
- solutionDeltaVertex.setInvokableClass(IterationTailPactTask.class);
-
- solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
-
- // tell the head that it needs to wait for the solution set updates
- headConfig.setWaitForSolutionSetUpdate();
- }
- else {
- // no tail, intermediate update. must be immediate update
- if (!iterNode.isImmediateSolutionSetUpdate()) {
- throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates.");
- }
- solutionDeltaConfig.setIsSolutionSetUpdateWithoutReprobe();
- }
- }
- }
-
- // ------------------- register the aggregators -------------------
- AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
- Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
-
- for (AggregatorWithName<?> agg : allAggregators) {
- if (agg.getName().equals(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME)) {
- throw new CompilerException("User defined aggregator used the same name as built-in workset " +
- "termination check aggregator: " + WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
- }
- }
-
- headConfig.addIterationAggregators(allAggregators);
- syncConfig.addIterationAggregators(allAggregators);
-
- String convAggName = aggs.getConvergenceCriterionAggregatorName();
- ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
-
- if (convCriterion != null || convAggName != null) {
- throw new CompilerException("Error: Cannot use custom convergence criterion with workset iteration. Workset iterations have implicit convergence criterion where workset is empty.");
- }
-
- headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
- syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
- syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
- }
-
- private static String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
- try {
- if (wrapper.hasObject()) {
- try {
- return wrapper.getUserCodeObject().toString();
- }
- catch (Throwable t) {
- return wrapper.getUserCodeClass().getName();
- }
- }
- else {
- return wrapper.getUserCodeClass().getName();
- }
- }
- catch (Throwable t) {
- return null;
- }
- }
-
- // -------------------------------------------------------------------------------------
- // Descriptors for tasks / configurations that are chained or merged with other tasks
- // -------------------------------------------------------------------------------------
-
- /**
- * Utility class that describes a task in a sequence of chained tasks. Chained tasks are tasks that run
- * together in one thread.
- */
- private static final class TaskInChain {
-
- private final Class<? extends ChainedDriver<?, ?>> chainedTask;
-
- private final TaskConfig taskConfig;
-
- private final String taskName;
-
- private AbstractJobVertex containingVertex;
-
- TaskInChain(Class<? extends ChainedDriver<?, ?>> chainedTask, TaskConfig taskConfig,
- String taskName) {
- this.chainedTask = chainedTask;
- this.taskConfig = taskConfig;
- this.taskName = taskName;
- }
-
- public Class<? extends ChainedDriver<?, ?>> getChainedTask() {
- return this.chainedTask;
- }
-
- public TaskConfig getTaskConfig() {
- return this.taskConfig;
- }
-
- public String getTaskName() {
- return this.taskName;
- }
-
- public AbstractJobVertex getContainingVertex() {
- return this.containingVertex;
- }
-
- public void setContainingVertex(AbstractJobVertex containingVertex) {
- this.containingVertex = containingVertex;
- }
- }
-
- private static final class IterationDescriptor {
-
- private final IterationPlanNode iterationNode;
-
- private AbstractJobVertex headTask;
-
- private TaskConfig headConfig;
-
- private TaskConfig headFinalResultConfig;
-
- private final int id;
-
- public IterationDescriptor(IterationPlanNode iterationNode, int id) {
- this.iterationNode = iterationNode;
- this.id = id;
- }
-
- public IterationPlanNode getIterationNode() {
- return iterationNode;
- }
-
- public void setHeadTask(AbstractJobVertex headTask, TaskConfig headConfig) {
- this.headTask = headTask;
- this.headFinalResultConfig = new TaskConfig(new Configuration());
-
- // check if we already had a configuration, for example if the solution set was
- if (this.headConfig != null) {
- headConfig.getConfiguration().addAll(this.headConfig.getConfiguration());
- }
-
- this.headConfig = headConfig;
- }
-
- public AbstractJobVertex getHeadTask() {
- return headTask;
- }
-
- public TaskConfig getHeadFinalResultConfig() {
- return headFinalResultConfig;
- }
-
- public int getId() {
- return this.id;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
deleted file mode 100644
index f2b736c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.postpass;
-
-import java.util.Map;
-
-public abstract class AbstractSchema<X> implements Iterable<Map.Entry<Integer, X>> {
-
- private int numConnectionsThatContributed;
-
-
- public int getNumConnectionsThatContributed() {
- return this.numConnectionsThatContributed;
- }
-
- public void increaseNumConnectionsThatContributed() {
- this.numConnectionsThatContributed++;
- }
-
- public abstract void addType(int pos, X type) throws ConflictingFieldTypeInfoException;
-
- public abstract X getType(int field);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
deleted file mode 100644
index 56d914c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.postpass;
-
-public final class ConflictingFieldTypeInfoException extends Exception {
-
- private static final long serialVersionUID = 3991352502693288321L;
-
- private final int fieldNumber;
-
- private final Object previousType, newType;
-
-
- public ConflictingFieldTypeInfoException(int fieldNumber, Object previousType, Object newType) {
- super("Conflicting type info for field " + fieldNumber + ": Old='" + previousType + "', new='" + newType + "'.");
- this.fieldNumber = fieldNumber;
- this.previousType = previousType;
- this.newType = newType;
- }
-
-
- public int getFieldNumber() {
- return fieldNumber;
- }
-
- public Object getPreviousType() {
- return this.previousType;
- }
-
- public Object getNewType() {
- return this.newType;
- }
-}
[22/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
deleted file mode 100644
index 9f2c467..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.operators.DualInputSemanticProperties;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.functions.SemanticPropUtil;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.types.ByteValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.junit.Test;
-
-
-public class RequestedGlobalPropertiesFilteringTest {
-
- private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
- new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
- );
-
- @Test(expected = NullPointerException.class)
- public void testNullProps() {
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0,1,2));
-
- rgProps.filterBySemanticProperties(null, 0);
- }
-
- @Test
- public void testEraseAll1() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0,1,2));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testEraseAll2() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"3;4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0, 1, 2));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testHashPartitioningPreserved1() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;3;4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
- assertNotNull(filtered.getPartitionedFields());
- assertEquals(3, filtered.getPartitionedFields().size());
- assertTrue(filtered.getPartitionedFields().contains(0));
- assertTrue(filtered.getPartitionedFields().contains(3));
- assertTrue(filtered.getPartitionedFields().contains(4));
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testHashPartitioningPreserved2() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
- assertNotNull(filtered.getPartitionedFields());
- assertEquals(3, filtered.getPartitionedFields().size());
- assertTrue(filtered.getPartitionedFields().contains(1));
- assertTrue(filtered.getPartitionedFields().contains(2));
- assertTrue(filtered.getPartitionedFields().contains(7));
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testHashPartitioningErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testAnyPartitioningPreserved1() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;3;4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
- assertNotNull(filtered.getPartitionedFields());
- assertEquals(3, filtered.getPartitionedFields().size());
- assertTrue(filtered.getPartitionedFields().contains(0));
- assertTrue(filtered.getPartitionedFields().contains(3));
- assertTrue(filtered.getPartitionedFields().contains(4));
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testAnyPartitioningPreserved2() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
- assertNotNull(filtered.getPartitionedFields());
- assertEquals(3, filtered.getPartitionedFields().size());
- assertTrue(filtered.getPartitionedFields().contains(1));
- assertTrue(filtered.getPartitionedFields().contains(2));
- assertTrue(filtered.getPartitionedFields().contains(7));
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testAnyPartitioningErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testRangePartitioningPreserved1() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;3;6"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(3, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setRangePartitioned(o);
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
- assertNotNull(filtered.getOrdering());
- assertEquals(3, filtered.getOrdering().getNumberOfFields());
- assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
- assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
- assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
- assertEquals(LongValue.class, filtered.getOrdering().getType(0));
- assertEquals(IntValue.class, filtered.getOrdering().getType(1));
- assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
- assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
- assertNull(filtered.getPartitionedFields());
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- }
-
- @Test
- public void testRangePartitioningPreserved2() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(3, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setRangePartitioned(o);
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
- assertNotNull(filtered.getOrdering());
- assertEquals(3, filtered.getOrdering().getNumberOfFields());
- assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
- assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
- assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
- assertEquals(LongValue.class, filtered.getOrdering().getType(0));
- assertEquals(IntValue.class, filtered.getOrdering().getType(1));
- assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
- assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
- assertNull(filtered.getPartitionedFields());
- assertNull(filtered.getDataDistribution());
- assertNull(filtered.getCustomPartitioner());
- }
-
- @Test
- public void testRangePartitioningPreserved3() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);
-
- DataDistribution dd = new MockDistribution();
- Ordering o = new Ordering();
- o.appendOrdering(3, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setRangePartitioned(o, dd);
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNotNull(filtered);
- assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
- assertNotNull(filtered.getOrdering());
- assertEquals(3, filtered.getOrdering().getNumberOfFields());
- assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
- assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
- assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
- assertEquals(LongValue.class, filtered.getOrdering().getType(0));
- assertEquals(IntValue.class, filtered.getOrdering().getType(1));
- assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
- assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
- assertNotNull(filtered.getDataDistribution());
- assertEquals(dd, filtered.getDataDistribution());
- assertNull(filtered.getPartitionedFields());
- assertNull(filtered.getCustomPartitioner());
- }
-
- @Test
- public void testRangePartitioningErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(3, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setRangePartitioned(o);
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testCustomPartitioningErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setCustomPartitioned(new FieldSet(0, 1, 2), new MockPartitioner());
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testRandomDistributionErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setRandomPartitioning();
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testReplicationErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setFullyReplicated();
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testRebalancingErased() {
-
- SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
- rgProps.setForceRebalancing();
-
- RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testDualHashPartitioningPreserved() {
-
- DualInputSemanticProperties dprops = new DualInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"0;2;4"}, new String[]{"1->3;4->6;3->7"},
- null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties gprops1 = new RequestedGlobalProperties();
- RequestedGlobalProperties gprops2 = new RequestedGlobalProperties();
- gprops1.setHashPartitioned(new FieldSet(2, 0, 4));
- gprops2.setHashPartitioned(new FieldSet(3, 6, 7));
- RequestedGlobalProperties filtered1 = gprops1.filterBySemanticProperties(dprops, 0);
- RequestedGlobalProperties filtered2 = gprops2.filterBySemanticProperties(dprops, 1);
-
- assertNotNull(filtered1);
- assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered1.getPartitioning());
- assertNotNull(filtered1.getPartitionedFields());
- assertEquals(3, filtered1.getPartitionedFields().size());
- assertTrue(filtered1.getPartitionedFields().contains(0));
- assertTrue(filtered1.getPartitionedFields().contains(2));
- assertTrue(filtered1.getPartitionedFields().contains(4));
- assertNull(filtered1.getOrdering());
- assertNull(filtered1.getCustomPartitioner());
- assertNull(filtered1.getDataDistribution());
-
- assertNotNull(filtered2);
- assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered2.getPartitioning());
- assertNotNull(filtered2.getPartitionedFields());
- assertEquals(3, filtered2.getPartitionedFields().size());
- assertTrue(filtered2.getPartitionedFields().contains(1));
- assertTrue(filtered2.getPartitionedFields().contains(3));
- assertTrue(filtered2.getPartitionedFields().contains(4));
- assertNull(filtered2.getOrdering());
- assertNull(filtered2.getCustomPartitioner());
- assertNull(filtered2.getDataDistribution());
- }
-
- @Test(expected = IndexOutOfBoundsException.class)
- public void testInvalidInputIndex() {
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- RequestedGlobalProperties gprops = new RequestedGlobalProperties();
- gprops.setHashPartitioned(new FieldSet(0,1));
-
- gprops.filterBySemanticProperties(sprops, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
deleted file mode 100644
index 0cede0e..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.operators.DualInputSemanticProperties;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.functions.SemanticPropUtil;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.types.ByteValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.junit.Test;
-
-public class RequestedLocalPropertiesFilteringTest {
-
- private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
- new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
- );
-
- @Test(expected = NullPointerException.class)
- public void testNullProps() {
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(0, 2, 3));
-
- rlProp.filterBySemanticProperties(null, 0);
- }
-
- @Test
- public void testAllErased() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(0, 2, 3));
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testGroupingPreserved1() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(0, 2, 3));
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNotNull(filtered);
- assertNotNull(filtered.getGroupedFields());
- assertEquals(3, filtered.getGroupedFields().size());
- assertTrue(filtered.getGroupedFields().contains(0));
- assertTrue(filtered.getGroupedFields().contains(2));
- assertTrue(filtered.getGroupedFields().contains(3));
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testGroupingPreserved2() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"3->0;5->2;1->3"}, null, null, tupleInfo, tupleInfo);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(0, 2, 3));
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNotNull(filtered);
- assertNotNull(filtered.getGroupedFields());
- assertEquals(3, filtered.getGroupedFields().size());
- assertTrue(filtered.getGroupedFields().contains(3));
- assertTrue(filtered.getGroupedFields().contains(5));
- assertTrue(filtered.getGroupedFields().contains(1));
- assertNull(filtered.getOrdering());
- }
-
- @Test
- public void testGroupingErased() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(0, 2, 3));
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testOrderPreserved1() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1;4;6"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(4, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setOrdering(o);
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNotNull(filtered);
- assertNotNull(filtered.getOrdering());
- assertEquals(3, filtered.getOrdering().getNumberOfFields());
- assertEquals(4, filtered.getOrdering().getFieldNumber(0).intValue());
- assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
- assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
- assertEquals(LongValue.class, filtered.getOrdering().getType(0));
- assertEquals(IntValue.class, filtered.getOrdering().getType(1));
- assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
- assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
- assertNull(filtered.getGroupedFields());
- }
-
- @Test
- public void testOrderPreserved2() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"5->1;0->4;2->6"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(4, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setOrdering(o);
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNotNull(filtered);
- assertNotNull(filtered.getOrdering());
- assertEquals(3, filtered.getOrdering().getNumberOfFields());
- assertEquals(0, filtered.getOrdering().getFieldNumber(0).intValue());
- assertEquals(5, filtered.getOrdering().getFieldNumber(1).intValue());
- assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
- assertEquals(LongValue.class, filtered.getOrdering().getType(0));
- assertEquals(IntValue.class, filtered.getOrdering().getType(1));
- assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
- assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
- assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
- assertNull(filtered.getGroupedFields());
- }
-
- @Test
- public void testOrderErased() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(4, LongValue.class, Order.DESCENDING);
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setOrdering(o);
-
- RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
-
- assertNull(filtered);
- }
-
- @Test
- public void testDualGroupingPreserved() {
-
- DualInputSemanticProperties dprops = new DualInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"},
- null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
-
- RequestedLocalProperties lprops1 = new RequestedLocalProperties();
- lprops1.setGroupedFields(new FieldSet(0,3,4));
-
- RequestedLocalProperties lprops2 = new RequestedLocalProperties();
- lprops2.setGroupedFields(new FieldSet(7, 1));
-
- RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
- RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);
-
- assertNotNull(filtered1);
- assertNotNull(filtered1.getGroupedFields());
- assertEquals(3, filtered1.getGroupedFields().size());
- assertTrue(filtered1.getGroupedFields().contains(1));
- assertTrue(filtered1.getGroupedFields().contains(2));
- assertTrue(filtered1.getGroupedFields().contains(3));
- assertNull(filtered1.getOrdering());
-
- assertNotNull(filtered2);
- assertNotNull(filtered2.getGroupedFields());
- assertEquals(2, filtered2.getGroupedFields().size());
- assertTrue(filtered2.getGroupedFields().contains(0));
- assertTrue(filtered2.getGroupedFields().contains(1));
- assertNull(filtered2.getOrdering());
- }
-
- @Test(expected = IndexOutOfBoundsException.class)
- public void testInvalidInputIndex() {
-
- SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);
-
- RequestedLocalProperties rlProp = new RequestedLocalProperties();
- rlProp.setGroupedFields(new FieldSet(1, 4));
-
- rlProp.filterBySemanticProperties(sProps, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
deleted file mode 100644
index b359e6b..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.CompilerTestBase;
-
-
-@SuppressWarnings({"serial", "unchecked"})
-public class DeltaIterationDependenciesTest extends CompilerTestBase {
-
- @Test
- public void testExceptionWhenNewWorksetNotDependentOnWorkset() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> deltaIteration = input.iterateDelta(input, 10,0);
-
- DataSet<Tuple2<Long, Long>> delta = deltaIteration.getSolutionSet().join(deltaIteration.getWorkset())
- .where(0).equalTo(0)
- .projectFirst(1).projectSecond(1);
-
- DataSet<Tuple2<Long, Long>> nextWorkset = deltaIteration.getSolutionSet().join(input)
- .where(0).equalTo(0)
- .projectFirst(1).projectSecond(1);
-
-
- DataSet<Tuple2<Long, Long>> result = deltaIteration.closeWith(delta, nextWorkset);
-
- result.print();
-
- Plan p = env.createProgramPlan();
- try {
- compileNoStats(p);
- fail("Should not be able to compile, since the next workset does not depend on the workset");
- }
- catch (CompilerException e) {
- // good
- }
- catch (Exception e) {
- fail("wrong exception type");
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
deleted file mode 100644
index de02836..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-@SuppressWarnings("serial")
-public class DistinctAndGroupingOptimizerTest extends CompilerTestBase {
-
- @Test
- public void testDistinctPreservesPartitioningOfDistinctFields() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(4);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L), new Tuple2<Long, Long>(1L, 1L))
- .map(new IdentityMapper<Tuple2<Long,Long>>()).setParallelism(4);
-
- data.distinct(0)
- .groupBy(0)
- .sum(1)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode distinctReducer = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
-
- // reducer can be forward, reuses partitioning from distinct
- assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
-
- // distinct reducer is partitioned
- assertEquals(ShipStrategyType.PARTITION_HASH, distinctReducer.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testDistinctDestroysPartitioningOfNonDistinctFields() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(4);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L), new Tuple2<Long, Long>(1L, 1L))
- .map(new IdentityMapper<Tuple2<Long,Long>>()).setParallelism(4);
-
- data.distinct(1)
- .groupBy(0)
- .sum(1)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
- SingleInputPlanNode distinctReducer = (SingleInputPlanNode) combiner.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
-
- // reducer must repartition, because it works on a different field
- assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
-
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
-
- // distinct reducer is partitioned
- assertEquals(ShipStrategyType.PARTITION_HASH, distinctReducer.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
deleted file mode 100644
index a683968..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.GroupReduceOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-
-import static org.junit.Assert.*;
-
-@SuppressWarnings("serial")
-public class GroupReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {
-
- @Test
- public void testAllGroupReduceNoCombiner() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
-
- data.reduceGroup(new RichGroupReduceFunction<Double, Double>() {
- public void reduce(Iterable<Double> values, Collector<Double> out) {}
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
-
- // the all-reduce has no combiner, when the DOP of the input is one
-
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // check wiring
- assertEquals(sourceNode, reduceNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that reduce has the right strategy
- assertEquals(DriverStrategy.ALL_GROUP_REDUCE, reduceNode.getDriverStrategy());
-
- // check DOP
- assertEquals(1, sourceNode.getParallelism());
- assertEquals(1, reduceNode.getParallelism());
- assertEquals(1, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testAllReduceWithCombiner() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
-
- GroupReduceOperator<Long, Long> reduced = data.reduceGroup(new RichGroupReduceFunction<Long, Long>() {
- public void reduce(Iterable<Long> values, Collector<Long> out) {}
- }).name("reducer");
-
- reduced.setCombinable(true);
- reduced.print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.ALL_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.ALL_GROUP_REDUCE_COMBINE, combineNode.getDriverStrategy());
-
- // check DOP
- assertEquals(8, sourceNode.getParallelism());
- assertEquals(8, combineNode.getParallelism());
- assertEquals(1, reduceNode.getParallelism());
- assertEquals(1, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
-
- @Test
- public void testGroupedReduceWithFieldPositionKeyNonCombinable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .groupBy(1)
- .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // check wiring
- assertEquals(sourceNode, reduceNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(1), reduceNode.getKeys(0));
- assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testGroupedReduceWithFieldPositionKeyCombinable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
- .groupBy(1)
- .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
- }).name("reducer");
-
- reduced.setCombinable(true);
- reduced.print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(1), reduceNode.getKeys(0));
- assertEquals(new FieldList(1), combineNode.getKeys(0));
- assertEquals(new FieldList(1), combineNode.getKeys(1));
- assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testGroupedReduceWithSelectorFunctionKeyNoncombinable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
- public String getKey(Tuple2<String, Double> value) { return value.f0; }
- })
- .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the key extractors and projectors
- SingleInputPlanNode keyExtractor = (SingleInputPlanNode) reduceNode.getInput().getSource();
- SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, keyExtractor.getInput().getSource());
- assertEquals(keyProjector, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(0), reduceNode.getKeys(0));
- assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, keyExtractor.getParallelism());
-
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, keyProjector.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testGroupedReduceWithSelectorFunctionKeyCombinable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
- .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
- public String getKey(Tuple2<String, Double> value) { return value.f0; }
- })
- .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
- }).name("reducer");
-
- reduced.setCombinable(true);
- reduced.print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // get the key extractors and projectors
- SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
- SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, keyExtractor.getInput().getSource());
- assertEquals(keyProjector, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(0), reduceNode.getKeys(0));
- assertEquals(new FieldList(0), combineNode.getKeys(0));
- assertEquals(new FieldList(0), combineNode.getKeys(1));
- assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, keyExtractor.getParallelism());
- assertEquals(6, combineNode.getParallelism());
-
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, keyProjector.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
deleted file mode 100644
index 37a8e81..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class IterationCompilerTest extends CompilerTestBase {
-
- @Test
- public void testIdentityIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(43);
-
- IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
- iteration.closeWith(iteration).print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testEmptyWorksetIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(43);
-
- DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
- .map(new MapFunction<Long, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Long value){ return null; }
- });
-
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
- iter.closeWith(iter.getWorkset(), iter.getWorkset())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testIterationWithUnionRoot() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(43);
-
- IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
-
- iteration.closeWith(
- iteration.map(new IdentityMapper<Long>()).union(iteration.map(new IdentityMapper<Long>())))
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- BulkIterationPlanNode iterNode = (BulkIterationPlanNode) sink.getInput().getSource();
-
- // make sure that the root is part of the dynamic path
-
- // the "NoOp" that comes after the union.
- SingleInputPlanNode noop = (SingleInputPlanNode) iterNode.getRootOfStepFunction();
- NAryUnionPlanNode union = (NAryUnionPlanNode) noop.getInput().getSource();
-
- assertTrue(noop.isOnDynamicPath());
- assertTrue(noop.getCostWeight() >= 1);
-
- assertTrue(union.isOnDynamicPath());
- assertTrue(union.getCostWeight() >= 1);
-
- // see that the jobgraph generator can translate this
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testWorksetIterationWithUnionRoot() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(43);
-
- DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
- .map(new MapFunction<Long, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Long value){ return null; }
- });
-
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
- iter.closeWith(
- iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
- .union(
- iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
- , iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
- .union(
- iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
- )
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
-
- // make sure that the root is part of the dynamic path
-
- // the "NoOp"a that come after the union.
- SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
- SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
-
- NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
- NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
-
- assertTrue(nextWorksetNoop.isOnDynamicPath());
- assertTrue(nextWorksetNoop.getCostWeight() >= 1);
-
- assertTrue(solutionDeltaNoop.isOnDynamicPath());
- assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
-
- assertTrue(nextWorksetUnion.isOnDynamicPath());
- assertTrue(nextWorksetUnion.getCostWeight() >= 1);
-
- assertTrue(solutionDeltaUnion.isOnDynamicPath());
- assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
deleted file mode 100644
index 0a62132..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class JoinTranslationTest extends CompilerTestBase {
-
- @Test
- public void testBroadcastHashFirstTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST);
- assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy());
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
- @Test
- public void testBroadcastHashSecondTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_SECOND);
- assertEquals(ShipStrategyType.FORWARD, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.BROADCAST, node.getInput2().getShipStrategy());
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
- @Test
- public void testPartitionHashFirstTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_HASH_FIRST);
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
- @Test
- public void testPartitionHashSecondTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_HASH_SECOND);
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
- @Test
- public void testPartitionSortMergeTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_SORT_MERGE);
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
- assertEquals(DriverStrategy.MERGE, node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
- @Test
- public void testOptimizerChoosesTest() {
- try {
- DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
- assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() ||
- DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + ": " + e.getMessage());
- }
- }
-
-
- private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> i1 = env.generateSequence(1, 1000);
- DataSet<Long> i2 = env.generateSequence(1, 1000);
-
- i1.join(i2, hint).where(new IdentityKeySelector<Long>()).equalTo(new IdentityKeySelector<Long>()).print();
-
- Plan plan = env.createProgramPlan();
-
- // set statistics to the sources
- plan.accept(new Visitor<Operator<?>>() {
- @Override
- public boolean preVisit(Operator<?> visitable) {
- if (visitable instanceof GenericDataSourceBase) {
- GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
- setSourceStatistics(source, 10000000, 1000);
- }
-
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> visitable) {}
- });
-
- OptimizedPlan op = compileWithStats(plan);
-
- return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
- }
-
-
-
- private static final class IdentityKeySelector<T> implements KeySelector<T, T> {
-
- @Override
- public T getKey(T value) {
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
deleted file mode 100644
index cd63b72..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class OpenIterationTest extends CompilerTestBase {
-
- @Test
- public void testSinkInOpenBulkIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> input = env.generateSequence(1, 10);
-
- IterativeDataSet<Long> iteration = input.iterate(10);
-
- DataSet<Long> mapped = iteration.map(new IdentityMapper<Long>());
-
- mapped.print();
-
- try {
- env.createProgramPlan();
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSinkInClosedBulkIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> input = env.generateSequence(1, 10);
-
- IterativeDataSet<Long> iteration = input.iterate(10);
-
- DataSet<Long> mapped = iteration.map(new IdentityMapper<Long>());
-
- iteration.closeWith(mapped).print();
-
- mapped.print();
-
- Plan p = env.createProgramPlan();
-
- try {
- compileNoStats(p);
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSinkOnSolutionSetDeltaIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
-
- DataSet<Tuple2<Long, Long>> mapped = iteration.getSolutionSet().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- mapped.print();
-
- try {
- env.createProgramPlan();
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSinkOnWorksetDeltaIteration() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
-
- DataSet<Tuple2<Long, Long>> mapped = iteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- mapped.print();
-
- try {
- env.createProgramPlan();
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testOperationOnSolutionSet() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
-
- DataSet<Tuple2<Long, Long>> mapped = iteration.getSolutionSet().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined = iteration.getWorkset().join(mapped)
- .where(0).equalTo(0).projectFirst(1).projectSecond(0);
-
- iteration.closeWith(joined, joined)
- .print();
-
- Plan p = env.createProgramPlan();
- try {
- compileNoStats(p);
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
deleted file mode 100644
index 8bb9a76..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import static org.junit.Assert.*;
-
-import java.util.Collections;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class PartitionOperatorTest extends CompilerTestBase {
-
- @Test
- public void testPartitionOperatorPreservesFields() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<Long, Long>(0L, 0L)));
-
- data.partitionCustom(new Partitioner<Long>() {
- public int partition(Long key, int numPartitions) { return key.intValue(); }
- }, 1)
- .groupBy(1)
- .reduceGroup(new IdentityGroupReducer<Tuple2<Long,Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode partitioner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
deleted file mode 100644
index 0724a9f..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.java;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-import static org.junit.Assert.*;
-
-@SuppressWarnings("serial")
-public class ReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {
-
- @Test
- public void testAllReduceNoCombiner() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
-
- data.reduce(new RichReduceFunction<Double>() {
-
- @Override
- public Double reduce(Double value1, Double value2){
- return value1 + value2;
- }
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
-
- // the all-reduce has no combiner, when the DOP of the input is one
-
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // check wiring
- assertEquals(sourceNode, reduceNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check DOP
- assertEquals(1, sourceNode.getParallelism());
- assertEquals(1, reduceNode.getParallelism());
- assertEquals(1, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testAllReduceWithCombiner() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
-
- data.reduce(new RichReduceFunction<Long>() {
-
- @Override
- public Long reduce(Long value1, Long value2){
- return value1 + value2;
- }
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.ALL_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.ALL_REDUCE, combineNode.getDriverStrategy());
-
- // check DOP
- assertEquals(8, sourceNode.getParallelism());
- assertEquals(8, combineNode.getParallelism());
- assertEquals(1, reduceNode.getParallelism());
- assertEquals(1, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testGroupedReduceWithFieldPositionKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .groupBy(1)
- .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
- @Override
- public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
- return null;
- }
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(1), reduceNode.getKeys(0));
- assertEquals(new FieldList(1), combineNode.getKeys(0));
- assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testGroupedReduceWithSelectorFunctionKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
- public String getKey(Tuple2<String, Double> value) { return value.f0; }
- })
- .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
- @Override
- public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
- return null;
- }
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // get the key extractors and projectors
- SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
- SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, keyExtractor.getInput().getSource());
- assertEquals(keyProjector, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(0), reduceNode.getKeys(0));
- assertEquals(new FieldList(0), combineNode.getKeys(0));
- assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, keyExtractor.getParallelism());
- assertEquals(6, combineNode.getParallelism());
-
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, keyProjector.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-}
[39/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/Costs.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/Costs.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/Costs.java
deleted file mode 100644
index 7c854bf..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/Costs.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.costs;
-
-/**
- * Simple class to represent the costs of an operation. The costs are currently tracking, network, I/O and CPU costs.
- *
- * Costs are composed of two parts of cost contributors:
- * <ol>
- * <li>Quantifiable costs. Those costs are used when estimates are available and track a quantifiable
- * measure, such as the number of bytes for network or I/O</li>
- * <li>Heuristic costs. Those costs are used when no estimates are available. They can be used to track that
- * an operator used a special operation which is heuristically considered more expensive than another
- * operation.</li>
- * </ol>
- * <p>
- * The quantifiable costs may frequently be unknown, which is represented by a {@code -1} as a value for the unknown
- * components of the cost. In that case, all operations' costs are unknown and hence it is not decidable which
- * operation to favor during pruning. In that case, the heuristic costs should contain a value to make sure that
- * operators with different strategies are comparable, even in the absence of estimates. The heuristic
- * costs are hence the system's mechanism of realizing pruning heuristics that favor some operations over others.
- */
-public class Costs implements Comparable<Costs>, Cloneable {
-
- public static final double UNKNOWN = -1;
-
- private double networkCost; // network cost, in transferred bytes
-
- private double diskCost; // bytes to be written and read, in bytes
-
- private double cpuCost; // CPU costs
-
- private double heuristicNetworkCost;
-
- private double heuristicDiskCost;
-
- private double heuristicCpuCost;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Default constructor. Initializes all costs to 0;
- */
- public Costs() {
- }
-
- /**
- * Creates a new costs object using the given values for the network and storage cost.
- *
- * @param networkCost The network cost, in bytes to be transferred.
- * @param diskCost The cost for disk, in bytes to be written and read.
- */
- public Costs(double networkCost, double diskCost) {
- setNetworkCost(networkCost);
- setDiskCost(diskCost);
- }
-
- /**
- * Creates a new costs object using the given values for the network and storage cost.
- *
- * @param networkCost The network cost, in bytes to be transferred.
- * @param diskCost The cost for disk, in bytes to be written and read.
- * @param cpuCost The cost for CPU operations.
- */
- public Costs(double networkCost, double diskCost, double cpuCost) {
- setNetworkCost(networkCost);
- setDiskCost(diskCost);
- setCpuCost(cpuCost);
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the network cost.
- *
- * @return The network cost, in bytes to be transferred.
- */
- public double getNetworkCost() {
- return networkCost;
- }
-
- /**
- * Sets the network cost for this Costs object.
- *
- * @param bytes
- * The network cost to set, in bytes to be transferred.
- */
- public void setNetworkCost(double bytes) {
- if (bytes == UNKNOWN || bytes >= 0) {
- this.networkCost = bytes;
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- /**
- * Adds the costs for network to the current network costs
- * for this Costs object.
- *
- * @param bytes The network cost to add, in bytes to be transferred.
- */
- public void addNetworkCost(double bytes) {
- this.networkCost = (this.networkCost < 0 || bytes < 0) ? UNKNOWN : this.networkCost + bytes;
- }
-
- /**
- * Gets the costs for disk.
- *
- * @return The disk cost, in bytes to be written and read.
- */
- public double getDiskCost() {
- return diskCost;
- }
-
- /**
- * Sets the costs for disk for this Costs object.
- *
- * @param bytes The disk cost to set, in bytes to be written and read.
- */
- public void setDiskCost(double bytes) {
- if (bytes == UNKNOWN || bytes >= 0) {
- this.diskCost = bytes;
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- /**
- * Adds the costs for disk to the current disk costs
- * for this Costs object.
- *
- * @param bytes The disk cost to add, in bytes to be written and read.
- */
- public void addDiskCost(double bytes) {
- this.diskCost =
- (this.diskCost < 0 || bytes < 0) ? UNKNOWN : this.diskCost + bytes;
- }
-
- /**
- * Gets the cost for the CPU.
- *
- * @return The CPU Cost.
- */
- public double getCpuCost() {
- return this.cpuCost;
- }
-
- /**
- * Sets the cost for the CPU.
- *
- * @param cost The CPU Cost.
- */
- public void setCpuCost(double cost) {
- if (cost == UNKNOWN || cost >= 0) {
- this.cpuCost = cost;
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- /**
- * Adds the given CPU cost to the current CPU cost for this Costs object.
- *
- * @param cost The CPU cost to add.
- */
- public void addCpuCost(double cost) {
- this.cpuCost =
- (this.cpuCost < 0 || cost < 0) ? UNKNOWN : this.cpuCost + cost;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the heuristic network cost.
- *
- * @return The heuristic network cost, in bytes to be transferred.
- */
- public double getHeuristicNetworkCost() {
- return this.heuristicNetworkCost;
- }
-
- /**
- * Sets the heuristic network cost for this Costs object.
- *
- * @param cost The heuristic network cost to set, in bytes to be transferred.
- */
- public void setHeuristicNetworkCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicNetworkCost = cost;
- }
-
- /**
- * Adds the heuristic costs for network to the current heuristic network costs
- * for this Costs object.
- *
- * @param cost The heuristic network cost to add.
- */
- public void addHeuristicNetworkCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicNetworkCost += cost;
- // check for overflow
- if (this.heuristicNetworkCost < 0) {
- this.heuristicNetworkCost = Double.MAX_VALUE;
- }
- }
-
- /**
- * Gets the heuristic costs for disk.
- *
- * @return The heuristic disk cost.
- */
- public double getHeuristicDiskCost() {
- return this.heuristicDiskCost;
- }
-
- /**
- * Sets the heuristic costs for disk for this Costs object.
- *
- * @param cost The heuristic disk cost to set.
- */
- public void setHeuristicDiskCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicDiskCost = cost;
- }
-
- /**
- * Adds the heuristic costs for disk to the current heuristic disk costs
- * for this Costs object.
- *
- * @param cost The heuristic disk cost to add.
- */
- public void addHeuristicDiskCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicDiskCost += cost;
- // check for overflow
- if (this.heuristicDiskCost < 0) {
- this.heuristicDiskCost = Double.MAX_VALUE;
- }
- }
-
- /**
- * Gets the heuristic cost for the CPU.
- *
- * @return The heuristic CPU Cost.
- */
- public double getHeuristicCpuCost() {
- return this.heuristicCpuCost;
- }
-
- /**
- * Sets the heuristic cost for the CPU.
- *
- * @param cost The heuristic CPU Cost.
- */
- public void setHeuristicCpuCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicCpuCost = cost;
- }
-
- /**
- * Adds the given heuristic CPU cost to the current heuristic CPU cost for this Costs object.
- *
- * @param cost The heuristic CPU cost to add.
- */
- public void addHeuristicCpuCost(double cost) {
- if (cost <= 0) {
- throw new IllegalArgumentException("Heuristic costs must be positive.");
- }
- this.heuristicCpuCost += cost;
- // check for overflow
- if (this.heuristicCpuCost < 0) {
- this.heuristicCpuCost = Double.MAX_VALUE;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Adds the given costs to these costs. If for one of the different cost components (network, disk),
- * the costs are unknown, the resulting costs will be unknown.
- *
- * @param other The costs to add.
- */
- public void addCosts(Costs other) {
- // ---------- quantifiable costs ----------
- if (this.networkCost == UNKNOWN || other.networkCost == UNKNOWN) {
- this.networkCost = UNKNOWN;
- } else {
- this.networkCost += other.networkCost;
- }
-
- if (this.diskCost == UNKNOWN || other.diskCost == UNKNOWN) {
- this.diskCost = UNKNOWN;
- } else {
- this.diskCost += other.diskCost;
- }
-
- if (this.cpuCost == UNKNOWN || other.cpuCost == UNKNOWN) {
- this.cpuCost = UNKNOWN;
- } else {
- this.cpuCost += other.cpuCost;
- }
-
- // ---------- heuristic costs ----------
-
- this.heuristicNetworkCost += other.heuristicNetworkCost;
- this.heuristicDiskCost += other.heuristicDiskCost;
- this.heuristicCpuCost += other.heuristicCpuCost;
- }
-
- /**
- * Subtracts the given costs from these costs. If the given costs are unknown, then these costs are remain unchanged.
- *
- * @param other The costs to subtract.
- */
- public void subtractCosts(Costs other) {
- if (this.networkCost != UNKNOWN && other.networkCost != UNKNOWN) {
- this.networkCost -= other.networkCost;
- if (this.networkCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- }
- if (this.diskCost != UNKNOWN && other.diskCost != UNKNOWN) {
- this.diskCost -= other.diskCost;
- if (this.diskCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- }
- if (this.cpuCost != UNKNOWN && other.cpuCost != UNKNOWN) {
- this.cpuCost -= other.cpuCost;
- if (this.cpuCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- }
-
- // ---------- relative costs ----------
-
- this.heuristicNetworkCost -= other.heuristicNetworkCost;
- if (this.heuristicNetworkCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- this.heuristicDiskCost -= other.heuristicDiskCost;
- if (this.heuristicDiskCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- this.heuristicCpuCost -= other.heuristicCpuCost;
- if (this.heuristicCpuCost < 0) {
- throw new IllegalArgumentException("Cannot subtract more cost then there is.");
- }
- }
-
- public void multiplyWith(int factor) {
- this.networkCost = this.networkCost < 0 ? -1 : this.networkCost * factor;
- this.diskCost = this.diskCost < 0 ? -1 : this.diskCost * factor;
- this.cpuCost = this.cpuCost < 0 ? -1 : this.cpuCost * factor;
- this.heuristicNetworkCost = this.heuristicNetworkCost < 0 ? -1 : this.heuristicNetworkCost * factor;
- this.heuristicDiskCost = this.heuristicDiskCost < 0 ? -1 : this.heuristicDiskCost * factor;
- this.heuristicCpuCost = this.heuristicCpuCost < 0 ? -1 : this.heuristicCpuCost * factor;
- }
-
- public void divideBy(int factor) {
- this.networkCost = this.networkCost < 0 ? -1 : this.networkCost / factor;
- this.diskCost = this.diskCost < 0 ? -1 : this.diskCost / factor;
- this.cpuCost = this.cpuCost < 0 ? -1 : this.cpuCost / factor;
- this.heuristicNetworkCost = this.heuristicNetworkCost < 0 ? -1 : this.heuristicNetworkCost / factor;
- this.heuristicDiskCost = this.heuristicDiskCost < 0 ? -1 : this.heuristicDiskCost / factor;
- this.heuristicCpuCost = this.heuristicCpuCost < 0 ? -1 : this.heuristicCpuCost / factor;
- }
-
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * The order of comparison is: network first, then disk, then CPU. The comparison here happens each time
- * primarily after the heuristic costs, then after the quantifiable costs.
- *
- * @see java.lang.Comparable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Costs o) {
- // check the network cost. if we have actual costs on both, use them, otherwise use the heuristic costs.
- if (this.networkCost != UNKNOWN && o.networkCost != UNKNOWN) {
- if (this.networkCost != o.networkCost) {
- return this.networkCost < o.networkCost ? -1 : 1;
- }
- } else if (this.heuristicNetworkCost < o.heuristicNetworkCost) {
- return -1;
- } else if (this.heuristicNetworkCost > o.heuristicNetworkCost) {
- return 1;
- }
-
- // next, check the disk cost. again, if we have actual costs on both, use them, otherwise use the heuristic costs.
- if (this.diskCost != UNKNOWN && o.diskCost != UNKNOWN) {
- if (this.diskCost != o.diskCost) {
- return this.diskCost < o.diskCost ? -1 : 1;
- }
- } else if (this.heuristicDiskCost < o.heuristicDiskCost) {
- return -1;
- } else if (this.heuristicDiskCost > o.heuristicDiskCost) {
- return 1;
- }
-
- // next, check the disk cost. again, if we have actual costs on both, use them, otherwise use the heuristic costs.
- if (this.cpuCost != UNKNOWN && o.cpuCost != UNKNOWN) {
- return this.cpuCost < o.cpuCost ? -1 : this.cpuCost > o.cpuCost ? 1 : 0;
- } else if (this.heuristicCpuCost < o.heuristicCpuCost) {
- return -1;
- } else if (this.heuristicCpuCost > o.heuristicCpuCost) {
- return 1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- long cpuCostBits = Double.doubleToLongBits(cpuCost);
- long heuristicCpuCostBits = Double.doubleToLongBits(heuristicCpuCost);
- long heuristicNetworkCostBits = Double.doubleToLongBits(heuristicNetworkCost);
- long heuristicDiskCostBits = Double.doubleToLongBits(heuristicDiskCost);
- long networkCostBits = Double.doubleToLongBits(networkCost);
- long diskCostBits = Double.doubleToLongBits(diskCost);
-
- result = prime * result + (int) (cpuCostBits ^ (cpuCostBits >>> 32));
- result = prime * result + (int) (heuristicCpuCostBits ^ (heuristicCpuCostBits >>> 32));
- result = prime * result + (int) (heuristicNetworkCostBits ^ (heuristicNetworkCostBits >>> 32));
- result = prime * result + (int) (heuristicDiskCostBits ^ (heuristicDiskCostBits >>> 32));
- result = prime * result + (int) (networkCostBits ^ (networkCostBits >>> 32));
- result = prime * result + (int) (diskCostBits ^ (diskCostBits >>> 32));
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == getClass()) {
- final Costs other = (Costs) obj;
- return this.networkCost == other.networkCost &
- this.diskCost == other.diskCost &
- this.cpuCost == other.cpuCost &
- this.heuristicNetworkCost == other.heuristicNetworkCost &
- this.heuristicDiskCost == other.heuristicDiskCost &
- this.heuristicCpuCost == other.heuristicCpuCost;
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "Costs [networkCost=" + networkCost + ", diskCost=" + diskCost +
- ", cpuCost=" + cpuCost + ", heuristicNetworkCost=" + heuristicNetworkCost +
- ", heuristicDiskCost=" + heuristicDiskCost + ", heuristicCpuCost=" + heuristicCpuCost + "]";
- }
-
- @Override
- public Costs clone() {
- try {
- return (Costs) super.clone();
- } catch (CloneNotSupportedException e) {
- throw new RuntimeException(e); // should never happen
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
deleted file mode 100644
index 543f330..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.costs;
-
-import org.apache.flink.optimizer.dag.EstimateProvider;
-
-/**
- * A default cost estimator that has access to basic size and cardinality estimates.
- * <p>
- * This estimator works with actual estimates (as far as they are available) and falls back to setting
- * relative costs, if no estimates are available. That way, the estimator makes sure that plans with
- * different strategies are costed differently, also in the absence of estimates. The different relative
- * costs in the absence of estimates represent this estimator's heuristic guidance towards certain strategies.
- * <p>
- * For robustness reasons, we always assume that the whole data is shipped during a repartition step. We deviate from
- * the typical estimate of <code>(n - 1) / n</code> (with <i>n</i> being the number of nodes), because for a parallelism
- * of 1, that would yield a shipping of zero bytes. While this is usually correct, the runtime scheduling may still
- * choose to move tasks to different nodes, so that we do not know that no data is shipped.
- */
-public class DefaultCostEstimator extends CostEstimator {
-
- /**
- * The case of the estimation for all relative costs. We heuristically pick a very large data volume, which
- * will favor strategies that are less expensive on large data volumes. This is robust and
- */
- private static final long HEURISTIC_COST_BASE = 1000000000L;
-
- // The numbers for the CPU effort are rather magic at the moment and should be seen rather ordinal
-
- private static final float MATERIALIZATION_CPU_FACTOR = 1;
-
- private static final float HASHING_CPU_FACTOR = 4;
-
- private static final float SORTING_CPU_FACTOR = 9;
-
-
- // --------------------------------------------------------------------------------------------
- // Shipping Strategy Cost
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void addRandomPartitioningCost(EstimateProvider estimates, Costs costs) {
- // conservative estimate: we need ship the whole data over the network to establish the
- // partitioning. no disk costs.
- final long estOutShipSize = estimates.getEstimatedOutputSize();
- if (estOutShipSize <= 0) {
- costs.setNetworkCost(Costs.UNKNOWN);
- } else {
- costs.addNetworkCost(estOutShipSize);
- }
- costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE);
- }
-
- @Override
- public void addHashPartitioningCost(EstimateProvider estimates, Costs costs) {
- // conservative estimate: we need ship the whole data over the network to establish the
- // partitioning. no disk costs.
- final long estOutShipSize = estimates.getEstimatedOutputSize();
- if (estOutShipSize <= 0) {
- costs.setNetworkCost(Costs.UNKNOWN);
- } else {
- costs.addNetworkCost(estOutShipSize);
- }
- costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE);
- }
-
- @Override
- public void addRangePartitionCost(EstimateProvider estimates, Costs costs) {
- final long dataSize = estimates.getEstimatedOutputSize();
- if (dataSize > 0) {
- // Assume sampling of 10% of the data and spilling it to disk
- final long sampled = (long) (dataSize * 0.1f);
- // set shipping costs
- costs.addNetworkCost(dataSize + sampled);
- } else {
- costs.setNetworkCost(Costs.UNKNOWN);
- }
-
- // no costs known. use the same assumption as above on the heuristic costs
- final long sampled = (long) (HEURISTIC_COST_BASE * 0.1f);
- costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE + sampled);
- costs.addHeuristicDiskCost(2 * sampled);
- }
-
- @Override
- public void addBroadcastCost(EstimateProvider estimates, int replicationFactor, Costs costs) {
- // if our replication factor is negative, we cannot calculate broadcast costs
- if (replicationFactor <= 0) {
- throw new IllegalArgumentException("The replication factor of must be larger than zero.");
- }
-
- if (replicationFactor > 0) {
- // assumption: we need ship the whole data over the network to each node.
- final long estOutShipSize = estimates.getEstimatedOutputSize();
- if (estOutShipSize <= 0) {
- costs.setNetworkCost(Costs.UNKNOWN);
- } else {
- costs.addNetworkCost(replicationFactor * estOutShipSize);
- }
- costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE * 10 * replicationFactor);
- } else {
- costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE * 1000);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Local Strategy Cost
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void addFileInputCost(long fileSizeInBytes, Costs costs) {
- if (fileSizeInBytes >= 0) {
- costs.addDiskCost(fileSizeInBytes);
- } else {
- costs.setDiskCost(Costs.UNKNOWN);
- }
- costs.addHeuristicDiskCost(HEURISTIC_COST_BASE);
- }
-
- @Override
- public void addLocalSortCost(EstimateProvider estimates, Costs costs) {
- final long s = estimates.getEstimatedOutputSize();
- // we assume a two phase merge sort, so all in all 2 I/O operations per block
- if (s <= 0) {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- } else {
- costs.addDiskCost(2 * s);
- costs.addCpuCost((long) (s * SORTING_CPU_FACTOR));
- }
- costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
- costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * SORTING_CPU_FACTOR));
- }
-
- @Override
- public void addLocalMergeCost(EstimateProvider input1, EstimateProvider input2, Costs costs, int costWeight) {
- // costs nothing. the very rarely incurred cost for a spilling block nested loops join in the
- // presence of massively re-occurring duplicate keys is ignored, because cannot be assessed
- }
-
- @Override
- public void addHybridHashCosts(EstimateProvider buildSideInput, EstimateProvider probeSideInput, Costs costs, int costWeight) {
- long bs = buildSideInput.getEstimatedOutputSize();
- long ps = probeSideInput.getEstimatedOutputSize();
-
- if (bs > 0 && ps > 0) {
- long overall = 2*bs + ps;
- costs.addDiskCost(overall);
- costs.addCpuCost((long) (overall * HASHING_CPU_FACTOR));
- } else {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- }
- costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
- costs.addHeuristicCpuCost((long) (2 * HEURISTIC_COST_BASE * HASHING_CPU_FACTOR));
-
- // cost weight applies to everything
- costs.multiplyWith(costWeight);
- }
-
- /**
- * Calculates the costs for the cached variant of the hybrid hash join.
- * We are assuming by default that half of the cached hash table fit into memory.
- */
- @Override
- public void addCachedHybridHashCosts(EstimateProvider buildSideInput, EstimateProvider probeSideInput, Costs costs, int costWeight) {
- if (costWeight < 1) {
- throw new IllegalArgumentException("The cost weight must be at least one.");
- }
-
- long bs = buildSideInput.getEstimatedOutputSize();
- long ps = probeSideInput.getEstimatedOutputSize();
-
- if (bs > 0 && ps > 0) {
- long overall = 2*bs + costWeight*ps;
- costs.addDiskCost(overall);
- costs.addCpuCost((long) (overall * HASHING_CPU_FACTOR));
- } else {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- }
-
- // one time the build side plus cost-weight time the probe side
- costs.addHeuristicDiskCost((1 + costWeight) * HEURISTIC_COST_BASE);
- costs.addHeuristicCpuCost((long) ((1 + costWeight) * HEURISTIC_COST_BASE * HASHING_CPU_FACTOR));
- }
-
- @Override
- public void addStreamedNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long bufferSize, Costs costs, int costWeight) {
- long is = innerSide.getEstimatedOutputSize();
- long oc = outerSide.getEstimatedNumRecords();
-
- if (is > 0 && oc >= 0) {
- // costs, if the inner side cannot be cached
- if (is > bufferSize) {
- costs.addDiskCost(oc * is);
- }
- costs.addCpuCost((long) (oc * is * MATERIALIZATION_CPU_FACTOR));
- } else {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- }
-
- // hack: assume 100k loops (should be expensive enough)
- costs.addHeuristicDiskCost(HEURISTIC_COST_BASE * 100000);
- costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * 100000 * MATERIALIZATION_CPU_FACTOR));
- costs.multiplyWith(costWeight);
- }
-
- @Override
- public void addBlockNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long blockSize, Costs costs, int costWeight) {
- long is = innerSide.getEstimatedOutputSize();
- long os = outerSide.getEstimatedOutputSize();
-
- if (is > 0 && os > 0) {
- long loops = Math.max(os / blockSize, 1);
- costs.addDiskCost(loops * is);
- costs.addCpuCost((long) (loops * is * MATERIALIZATION_CPU_FACTOR));
- } else {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- }
-
- // hack: assume 1k loops (much cheaper than the streamed variant!)
- costs.addHeuristicDiskCost(HEURISTIC_COST_BASE * 1000);
- costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * 1000 * MATERIALIZATION_CPU_FACTOR));
- costs.multiplyWith(costWeight);
- }
-
- // --------------------------------------------------------------------------------------------
- // Damming Cost
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void addArtificialDamCost(EstimateProvider estimates, long bufferSize, Costs costs) {
- final long s = estimates.getEstimatedOutputSize();
- // we assume spilling and re-reading
- if (s <= 0) {
- costs.setDiskCost(Costs.UNKNOWN);
- costs.setCpuCost(Costs.UNKNOWN);
- } else {
- costs.addDiskCost(2 * s);
- costs.setCpuCost((long) (s * MATERIALIZATION_CPU_FACTOR));
- }
- costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
- costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * MATERIALIZATION_CPU_FACTOR));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
deleted file mode 100644
index d199ae7..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.util.Visitor;
-
-/**
- * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
- */
-public abstract class AbstractPartialSolutionNode extends OptimizerNode {
-
- protected AbstractPartialSolutionNode(Operator<?> contract) {
- super(contract);
- }
-
- // --------------------------------------------------------------------------------------------
-
- protected void copyEstimates(OptimizerNode node) {
- this.estimatedNumRecords = node.estimatedNumRecords;
- this.estimatedOutputSize = node.estimatedOutputSize;
- }
-
- public abstract IterationNode getIterationNode();
-
- // --------------------------------------------------------------------------------------------
-
- public boolean isOnDynamicPath() {
- return true;
- }
-
- public void identifyDynamicPath(int costWeight) {
- this.onDynamicPath = true;
- this.costWeight = costWeight;
- }
-
- @Override
- public List<DagConnection> getIncomingConnections() {
- return Collections.emptyList();
- }
-
- @Override
- public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode dataExchangeMode) {}
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // we do nothing here, because the estimates can only be copied from the iteration input
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- // no children, so nothing to compute
- }
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- } else {
- throw new IllegalStateException();
- }
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new EmptySemanticProperties();
- }
-
- @Override
- protected void readStubAnnotations() {}
-
- @Override
- public void accept(Visitor<OptimizerNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
deleted file mode 100644
index 068799e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.Union;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.BinaryUnionOpDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-/**
- * The Optimizer representation of a binary <i>Union</i>.
- */
-public class BinaryUnionNode extends TwoInputNode {
-
- private Set<RequestedGlobalProperties> channelProps;
-
- public BinaryUnionNode(Union<?> union){
- super(union);
- }
-
- @Override
- public String getName() {
- return "Union";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return Collections.emptyList();
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
- long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
- this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 + card2;
-
- long size1 = getFirstPredecessorNode().getEstimatedOutputSize();
- long size2 = getSecondPredecessorNode().getEstimatedOutputSize();
- this.estimatedOutputSize = (size1 < 0 || size2 < 0) ? -1 : size1 + size2;
- }
-
- @Override
- public void computeUnionOfInterestingPropertiesFromSuccessors() {
- super.computeUnionOfInterestingPropertiesFromSuccessors();
- // clear all local properties, as they are destroyed anyways
- getInterestingProperties().getLocalProperties().clear();
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- final InterestingProperties props = getInterestingProperties();
-
- // if no other properties exist, add the pruned trivials back
- if (props.getGlobalProperties().isEmpty()) {
- props.addGlobalProperties(new RequestedGlobalProperties());
- }
- props.addLocalProperties(new RequestedLocalProperties());
- this.input1.setInterestingProperties(props.clone());
- this.input2.setInterestingProperties(props.clone());
-
- this.channelProps = props.getGlobalProperties();
- }
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- // check if we have a cached version
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- }
-
- // step down to all producer nodes and calculate alternative plans
- final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
- final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
-
- List<DagConnection> broadcastConnections = getBroadcastConnections();
- if (broadcastConnections != null && broadcastConnections.size() > 0) {
- throw new CompilerException("Found BroadcastVariables on a Union operation");
- }
-
- final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
-
- final List<Set<? extends NamedChannel>> broadcastPlanChannels = Collections.emptyList();
-
- final BinaryUnionOpDescriptor operator = new BinaryUnionOpDescriptor();
- final RequestedLocalProperties noLocalProps = new RequestedLocalProperties();
-
- final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
- final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
-
- final int dop = getParallelism();
- final int inDop1 = getFirstPredecessorNode().getParallelism();
- final int inDop2 = getSecondPredecessorNode().getParallelism();
-
- final boolean dopChange1 = dop != inDop1;
- final boolean dopChange2 = dop != inDop2;
-
- final boolean input1breakPipeline = this.input1.isBreakingPipeline();
- final boolean input2breakPipeline = this.input2.isBreakingPipeline();
-
- // enumerate all pairwise combination of the children's plans together with
- // all possible operator strategy combination
-
- // create all candidates
- for (PlanNode child1 : subPlans1) {
- for (PlanNode child2 : subPlans2) {
-
- // check that the children go together. that is the case if they build upon the same
- // candidate at the joined branch plan.
- if (!areBranchCompatible(child1, child2)) {
- continue;
- }
-
- for (RequestedGlobalProperties igps: this.channelProps) {
- // create a candidate channel for the first input. mark it cached, if the connection says so
- Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
- if (this.input1.getShipStrategy() == null) {
- // free to choose the ship strategy
- igps.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
-
- // if the DOP changed, make sure that we cancel out properties, unless the
- // ship strategy preserves/establishes them even under changing DOPs
- if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
- c1.getGlobalProperties().reset();
- }
- }
- else {
- // ship strategy fixed by compiler hint
- ShipStrategyType shipStrategy = this.input1.getShipStrategy();
- DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipStrategy, input1breakPipeline);
- if (this.keys1 != null) {
- c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList(), exMode);
- } else {
- c1.setShipStrategy(this.input1.getShipStrategy(), exMode);
- }
-
- if (dopChange1) {
- c1.adjustGlobalPropertiesForFullParallelismChange();
- }
- }
-
- // create a candidate channel for the first input. mark it cached, if the connection says so
- Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
- if (this.input2.getShipStrategy() == null) {
- // free to choose the ship strategy
- igps.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
-
- // if the DOP changed, make sure that we cancel out properties, unless the
- // ship strategy preserves/establishes them even under changing DOPs
- if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
- c2.getGlobalProperties().reset();
- }
- } else {
- // ship strategy fixed by compiler hint
- ShipStrategyType shipStrategy = this.input2.getShipStrategy();
- DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipStrategy, input2breakPipeline);
- if (this.keys2 != null) {
- c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList(), exMode);
- } else {
- c2.setShipStrategy(this.input2.getShipStrategy(), exMode);
- }
-
- if (dopChange2) {
- c2.adjustGlobalPropertiesForFullParallelismChange();
- }
- }
-
- // get the global properties and clear unique fields (not preserved anyways during the union)
- GlobalProperties p1 = c1.getGlobalProperties();
- GlobalProperties p2 = c2.getGlobalProperties();
- p1.clearUniqueFieldCombinations();
- p2.clearUniqueFieldCombinations();
-
- // adjust the partitioning, if they exist but are not equal. this may happen when both channels have a
- // partitioning that fulfills the requirements, but both are incompatible. For example may a property requirement
- // be ANY_PARTITIONING on fields (0) and one channel is range partitioned on that field, the other is hash
- // partitioned on that field.
- if (!igps.isTrivial() && !(p1.equals(p2))) {
- if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() != ShipStrategyType.FORWARD) {
- // adjust c2 to c1
- c2 = c2.clone();
- p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
- }
- else if (c2.getShipStrategy() == ShipStrategyType.FORWARD && c1.getShipStrategy() != ShipStrategyType.FORWARD) {
- // adjust c1 to c2
- c1 = c1.clone();
- p2.parameterizeChannel(c1,dopChange1, input1Mode, input1breakPipeline);
- }
- else if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() == ShipStrategyType.FORWARD) {
- boolean adjustC1 = c1.getEstimatedOutputSize() <= 0 || c2.getEstimatedOutputSize() <= 0 ||
- c1.getEstimatedOutputSize() <= c2.getEstimatedOutputSize();
- if (adjustC1) {
- c2 = c2.clone();
- p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
- } else {
- c1 = c1.clone();
- p2.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
- }
- } else {
- // this should never happen, as it implies both realize a different strategy, which is
- // excluded by the check that the required strategies must match
- throw new CompilerException("Bug in Plan Enumeration for Union Node.");
- }
- }
-
-
- instantiate(operator, c1, c2, broadcastPlanChannels, outputPlans, estimator, igps, igps, noLocalProps, noLocalProps);
- }
- }
- }
-
- // cost and prune the plans
- for (PlanNode node : outputPlans) {
- estimator.costOperator(node);
- }
- prunePlanAlternatives(outputPlans);
- outputPlans.trimToSize();
-
- this.cachedPlans = outputPlans;
- return outputPlans;
- }
-
- @Override
- protected void readStubAnnotations() {}
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new UnionSemanticProperties();
- }
-
- @Override
- public void computeOutputEstimates(DataStatistics statistics) {
- OptimizerNode in1 = getFirstPredecessorNode();
- OptimizerNode in2 = getSecondPredecessorNode();
-
- this.estimatedNumRecords = in1.estimatedNumRecords > 0 && in2.estimatedNumRecords > 0 ?
- in1.estimatedNumRecords + in2.estimatedNumRecords : -1;
- this.estimatedOutputSize = in1.estimatedOutputSize > 0 && in2.estimatedOutputSize > 0 ?
- in1.estimatedOutputSize + in2.estimatedOutputSize : -1;
- }
-
- public static class UnionSemanticProperties implements SemanticProperties {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public FieldSet getForwardingTargetFields(int input, int sourceField) {
- if (input != 0 && input != 1) {
- throw new IndexOutOfBoundsException("Invalid input index for binary union node.");
- }
-
- return new FieldSet(sourceField);
- }
-
- @Override
- public int getForwardingSourceField(int input, int targetField) {
- if (input != 0 && input != 1) {
- throw new IndexOutOfBoundsException();
- }
-
- return targetField;
- }
-
- @Override
- public FieldSet getReadFields(int input) {
- if (input != 0 && input != 1) {
- throw new IndexOutOfBoundsException();
- }
-
- return FieldSet.EMPTY_SET;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
deleted file mode 100644
index 55b8785..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dag.WorksetIterationNode.SingleRootJoiner;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.NoOpDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * A node in the optimizer's program representation for a bulk iteration.
- */
-public class BulkIterationNode extends SingleInputNode implements IterationNode {
-
- private BulkPartialSolutionNode partialSolution;
-
- private OptimizerNode terminationCriterion;
-
- private OptimizerNode nextPartialSolution;
-
- private DagConnection rootConnection; // connection out of the next partial solution
-
- private DagConnection terminationCriterionRootConnection; // connection out of the term. criterion
-
- private OptimizerNode singleRoot;
-
- private final int costWeight;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new node for the bulk iteration.
- *
- * @param iteration The bulk iteration the node represents.
- */
- public BulkIterationNode(BulkIterationBase<?> iteration) {
- super(iteration);
-
- if (iteration.getMaximumNumberOfIterations() <= 0) {
- throw new CompilerException("BulkIteration must have a maximum number of iterations specified.");
- }
-
- int numIters = iteration.getMaximumNumberOfIterations();
-
- this.costWeight = (numIters > 0 && numIters < OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) ?
- numIters : OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public BulkIterationBase<?> getIterationContract() {
- return (BulkIterationBase<?>) getOperator();
- }
-
- /**
- * Gets the partialSolution from this BulkIterationNode.
- *
- * @return The partialSolution.
- */
- public BulkPartialSolutionNode getPartialSolution() {
- return partialSolution;
- }
-
- /**
- * Sets the partialSolution for this BulkIterationNode.
- *
- * @param partialSolution The partialSolution to set.
- */
- public void setPartialSolution(BulkPartialSolutionNode partialSolution) {
- this.partialSolution = partialSolution;
- }
-
-
- /**
- * Gets the nextPartialSolution from this BulkIterationNode.
- *
- * @return The nextPartialSolution.
- */
- public OptimizerNode getNextPartialSolution() {
- return nextPartialSolution;
- }
-
- /**
- * Sets the nextPartialSolution for this BulkIterationNode.
- *
- * @param nextPartialSolution The nextPartialSolution to set.
- */
- public void setNextPartialSolution(OptimizerNode nextPartialSolution, OptimizerNode terminationCriterion) {
-
- // check if the root of the step function has the same DOP as the iteration
- // or if the step function has any operator at all
- if (nextPartialSolution.getParallelism() != getParallelism() ||
- nextPartialSolution == partialSolution || nextPartialSolution instanceof BinaryUnionNode)
- {
- // add a no-op to the root to express the re-partitioning
- NoOpNode noop = new NoOpNode();
- noop.setDegreeOfParallelism(getParallelism());
-
- DagConnection noOpConn = new DagConnection(nextPartialSolution, noop, ExecutionMode.PIPELINED);
- noop.setIncomingConnection(noOpConn);
- nextPartialSolution.addOutgoingConnection(noOpConn);
-
- nextPartialSolution = noop;
- }
-
- this.nextPartialSolution = nextPartialSolution;
- this.terminationCriterion = terminationCriterion;
-
- if (terminationCriterion == null) {
- this.singleRoot = nextPartialSolution;
- this.rootConnection = new DagConnection(nextPartialSolution, ExecutionMode.PIPELINED);
- }
- else {
- // we have a termination criterion
- SingleRootJoiner singleRootJoiner = new SingleRootJoiner();
- this.rootConnection = new DagConnection(nextPartialSolution, singleRootJoiner, ExecutionMode.PIPELINED);
- this.terminationCriterionRootConnection = new DagConnection(terminationCriterion, singleRootJoiner,
- ExecutionMode.PIPELINED);
-
- singleRootJoiner.setInputs(this.rootConnection, this.terminationCriterionRootConnection);
-
- this.singleRoot = singleRootJoiner;
-
- // add connection to terminationCriterion for interesting properties visitor
- terminationCriterion.addOutgoingConnection(terminationCriterionRootConnection);
-
- }
-
- nextPartialSolution.addOutgoingConnection(rootConnection);
- }
-
- public int getCostWeight() {
- return this.costWeight;
- }
-
- public OptimizerNode getSingleRootOfStepFunction() {
- return this.singleRoot;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public String getName() {
- return "Bulk Iteration";
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new EmptySemanticProperties();
- }
-
- protected void readStubAnnotations() {}
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- }
-
- // --------------------------------------------------------------------------------------------
- // Properties and Optimization
- // --------------------------------------------------------------------------------------------
-
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return Collections.<OperatorDescriptorSingle>singletonList(new NoOpDescriptor());
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- final InterestingProperties intProps = getInterestingProperties().clone();
-
- if (this.terminationCriterion != null) {
- // first propagate through termination Criterion. since it has no successors, it has no
- // interesting properties
- this.terminationCriterionRootConnection.setInterestingProperties(new InterestingProperties());
- this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator));
- }
-
- // we need to make 2 interesting property passes, because the root of the step function needs also
- // the interesting properties as generated by the partial solution
-
- // give our own interesting properties (as generated by the iterations successors) to the step function and
- // make the first pass
- this.rootConnection.setInterestingProperties(intProps);
- this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
-
- // take the interesting properties of the partial solution and add them to the root interesting properties
- InterestingProperties partialSolutionIntProps = this.partialSolution.getInterestingProperties();
- intProps.getGlobalProperties().addAll(partialSolutionIntProps.getGlobalProperties());
- intProps.getLocalProperties().addAll(partialSolutionIntProps.getLocalProperties());
-
- // clear all interesting properties to prepare the second traversal
- // this clears only the path down from the next partial solution. The paths down
- // from the termination criterion (before they meet the paths down from the next partial solution)
- // remain unaffected by this step
- this.rootConnection.clearInterestingProperties();
- this.nextPartialSolution.accept(InterestingPropertiesClearer.INSTANCE);
-
- // 2nd pass
- this.rootConnection.setInterestingProperties(intProps);
- this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
-
- // now add the interesting properties of the partial solution to the input
- final InterestingProperties inProps = this.partialSolution.getInterestingProperties().clone();
- inProps.addGlobalProperties(new RequestedGlobalProperties());
- inProps.addLocalProperties(new RequestedLocalProperties());
- this.inConn.setInterestingProperties(inProps);
- }
-
- @Override
- public void clearInterestingProperties() {
- super.clearInterestingProperties();
-
- this.singleRoot.accept(InterestingPropertiesClearer.INSTANCE);
- this.rootConnection.clearInterestingProperties();
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- // the resulting branches are those of the step function
- // because the BulkPartialSolution takes the input's branches
- addClosedBranches(getSingleRootOfStepFunction().closedBranchingNodes);
- List<UnclosedBranchDescriptor> result = getSingleRootOfStepFunction().openBranches;
-
- this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
- }
-
-
- @Override
- protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
- List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
- {
- // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
- // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
- // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
- // we have a fitting candidate for the step function (often, work is pushed out of the step function).
- // Among the candidates of the step function, we keep only those that meet the requested properties of the
- // current candidate initial partial solution. That makes sure these properties exist at the beginning of
- // the successive iteration.
-
- // 1) Because we enumerate multiple times, we may need to clean the cached plans
- // before starting another enumeration
- this.nextPartialSolution.accept(PlanCacheCleaner.INSTANCE);
- if (this.terminationCriterion != null) {
- this.terminationCriterion.accept(PlanCacheCleaner.INSTANCE);
- }
-
- // 2) Give the partial solution the properties of the current candidate for the initial partial solution
- this.partialSolution.setCandidateProperties(in.getGlobalProperties(), in.getLocalProperties(), in);
- final BulkPartialSolutionPlanNode pspn = this.partialSolution.getCurrentPartialSolutionPlanNode();
-
- // 3) Get the alternative plans
- List<PlanNode> candidates = this.nextPartialSolution.getAlternativePlans(estimator);
-
- // 4) Make sure that the beginning of the step function does not assume properties that
- // are not also produced by the end of the step function.
-
- {
- List<PlanNode> newCandidates = new ArrayList<PlanNode>();
-
- for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
- PlanNode candidate = planDeleter.next();
-
- GlobalProperties atEndGlobal = candidate.getGlobalProperties();
- LocalProperties atEndLocal = candidate.getLocalProperties();
-
- FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobal, atEndLocal);
- if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
- ; // depends only through broadcast variable on the partial solution
- }
- else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
- // attach a no-op node through which we create the properties of the original input
- Channel toNoOp = new Channel(candidate);
- globPropsReq.parameterizeChannel(toNoOp, false, rootConnection.getDataExchangeMode(), false);
- locPropsReq.parameterizeChannel(toNoOp);
-
- UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
- rebuildPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
-
- SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
- rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
- estimator.costOperator(rebuildPropertiesPlanNode);
-
- GlobalProperties atEndGlobalModified = rebuildPropertiesPlanNode.getGlobalProperties();
- LocalProperties atEndLocalModified = rebuildPropertiesPlanNode.getLocalProperties();
-
- if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
- FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobalModified, atEndLocalModified);
-
- if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
- newCandidates.add(rebuildPropertiesPlanNode);
- }
- }
-
- planDeleter.remove();
- }
- }
- }
-
- if (candidates.isEmpty()) {
- return;
- }
-
- // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
- if (terminationCriterion == null) {
- for (PlanNode candidate : candidates) {
- BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate);
- GlobalProperties gProps = candidate.getGlobalProperties().clone();
- LocalProperties lProps = candidate.getLocalProperties().clone();
- node.initProperties(gProps, lProps);
- target.add(node);
- }
- }
- else if (candidates.size() > 0) {
- List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);
-
- SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
-
- for (PlanNode candidate : candidates) {
- for (PlanNode terminationCandidate : terminationCriterionCandidates) {
- if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
- BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate);
- GlobalProperties gProps = candidate.getGlobalProperties().clone();
- LocalProperties lProps = candidate.getLocalProperties().clone();
- node.initProperties(gProps, lProps);
- target.add(node);
-
- }
- }
- }
-
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Iteration Specific Traversals
- // --------------------------------------------------------------------------------------------
-
- public void acceptForStepFunction(Visitor<OptimizerNode> visitor) {
- this.singleRoot.accept(visitor);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
deleted file mode 100644
index 25a7eef..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.BulkIterationBase.PartialSolutionPlaceHolder;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.PlanNode;
-
-/**
- * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
- */
-public class BulkPartialSolutionNode extends AbstractPartialSolutionNode {
-
- private final BulkIterationNode iterationNode;
-
-
- public BulkPartialSolutionNode(PartialSolutionPlaceHolder<?> psph, BulkIterationNode iterationNode) {
- super(psph);
- this.iterationNode = iterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
- if (this.cachedPlans != null) {
- throw new IllegalStateException();
- } else {
- this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
- "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
- }
- }
-
- public BulkPartialSolutionPlanNode getCurrentPartialSolutionPlanNode() {
- if (this.cachedPlans != null) {
- return (BulkPartialSolutionPlanNode) this.cachedPlans.get(0);
- } else {
- throw new IllegalStateException();
- }
- }
-
- public BulkIterationNode getIterationNode() {
- return this.iterationNode;
- }
-
- @Override
- public void computeOutputEstimates(DataStatistics statistics) {
- copyEstimates(this.iterationNode.getPredecessorNode());
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the operator (here the {@link PartialSolutionPlaceHolder}) that is represented by this
- * optimizer node.
- *
- * @return The operator represented by this optimizer node.
- */
- @Override
- public PartialSolutionPlaceHolder<?> getOperator() {
- return (PartialSolutionPlaceHolder<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Bulk Partial Solution";
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- OptimizerNode inputToIteration = this.iterationNode.getPredecessorNode();
-
- addClosedBranches(inputToIteration.closedBranchingNodes);
- List<UnclosedBranchDescriptor> fromInput = inputToIteration.getBranchesForParent(this.iterationNode.getIncomingConnection());
- this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
deleted file mode 100644
index 92076c3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.CoGroupDescriptor;
-import org.apache.flink.optimizer.operators.CoGroupWithSolutionSetFirstDescriptor;
-import org.apache.flink.optimizer.operators.CoGroupWithSolutionSetSecondDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-
-/**
- * The Optimizer representation of a <i>CoGroup</i> operator.
- */
-public class CoGroupNode extends TwoInputNode {
-
- private List<OperatorDescriptorDual> dataProperties;
-
- public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) {
- super(operator);
- this.dataProperties = initializeDataProperties(operator.getCustomPartitioner());
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the operator for this CoGroup node.
- *
- * @return The CoGroup operator.
- */
- @Override
- public CoGroupOperatorBase<?, ?, ?, ?> getOperator() {
- return (CoGroupOperatorBase<?, ?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "CoGroup";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return this.dataProperties;
- }
-
- public void makeCoGroupWithSolutionSet(int solutionsetInputIndex) {
- OperatorDescriptorDual op;
- if (solutionsetInputIndex == 0) {
- op = new CoGroupWithSolutionSetFirstDescriptor(keys1, keys2);
- } else if (solutionsetInputIndex == 1) {
- op = new CoGroupWithSolutionSetSecondDescriptor(keys1, keys2);
- } else {
- throw new IllegalArgumentException();
- }
- this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(op);
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // for CoGroup, we currently make no reasonable default estimates
- }
-
- private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) {
- Ordering groupOrder1 = null;
- Ordering groupOrder2 = null;
-
- CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
- groupOrder1 = cgc.getGroupOrderForInputOne();
- groupOrder2 = cgc.getGroupOrderForInputTwo();
-
- if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
- groupOrder1 = null;
- }
- if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
- groupOrder2 = null;
- }
-
- CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2);
- if (customPartitioner != null) {
- descr.setCustomPartitioner(customPartitioner);
- }
-
- return Collections.<OperatorDescriptorDual>singletonList(descr);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
deleted file mode 100644
index 93be1e4..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.CollectorMapDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-/**
- * The optimizer's internal representation of a <i>Map</i> operator node.
- */
-public class CollectorMapNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
-
- public CollectorMapNode(SingleInputOperator<?, ?, ?> operator) {
- super(operator);
-
- this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new CollectorMapDescriptor());
- }
-
- @Override
- public String getName() {
- return "Map";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- /**
- * Computes the estimates for the Map operator. Map takes one value and transforms it into another value.
- * The cardinality consequently stays the same.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
deleted file mode 100644
index 8de67e8..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.CrossOperatorBase;
-import org.apache.flink.api.common.operators.base.CrossOperatorBase.CrossHint;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.operators.CrossBlockOuterFirstDescriptor;
-import org.apache.flink.optimizer.operators.CrossBlockOuterSecondDescriptor;
-import org.apache.flink.optimizer.operators.CrossStreamOuterFirstDescriptor;
-import org.apache.flink.optimizer.operators.CrossStreamOuterSecondDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.configuration.Configuration;
-
-/**
- * The Optimizer representation of a <i>Cross</i> (Cartesian product) operator.
- */
-public class CrossNode extends TwoInputNode {
-
- private final List<OperatorDescriptorDual> dataProperties;
-
- /**
- * Creates a new CrossNode for the given operator.
- *
- * @param operation The Cross operator object.
- */
- public CrossNode(CrossOperatorBase<?, ?, ?, ?> operation) {
- super(operation);
-
- Configuration conf = operation.getParameters();
- String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
-
- CrossHint hint = operation.getCrossHint();
-
- if (localStrategy != null) {
-
- final boolean allowBCfirst = hint != CrossHint.SECOND_IS_SMALL;
- final boolean allowBCsecond = hint != CrossHint.FIRST_IS_SMALL;
-
- final OperatorDescriptorDual fixedDriverStrat;
- if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
- fixedDriverStrat = new CrossBlockOuterFirstDescriptor(allowBCfirst, allowBCsecond);
- } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND.equals(localStrategy)) {
- fixedDriverStrat = new CrossBlockOuterSecondDescriptor(allowBCfirst, allowBCsecond);
- } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST.equals(localStrategy)) {
- fixedDriverStrat = new CrossStreamOuterFirstDescriptor(allowBCfirst, allowBCsecond);
- } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND.equals(localStrategy)) {
- fixedDriverStrat = new CrossStreamOuterSecondDescriptor(allowBCfirst, allowBCsecond);
- } else {
- throw new CompilerException("Invalid local strategy hint for cross contract: " + localStrategy);
- }
-
- this.dataProperties = Collections.singletonList(fixedDriverStrat);
- }
- else if (hint == CrossHint.SECOND_IS_SMALL) {
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
- list.add(new CrossBlockOuterSecondDescriptor(false, true));
- list.add(new CrossStreamOuterFirstDescriptor(false, true));
- this.dataProperties = list;
- }
- else if (hint == CrossHint.FIRST_IS_SMALL) {
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
- list.add(new CrossBlockOuterFirstDescriptor(true, false));
- list.add(new CrossStreamOuterSecondDescriptor(true, false));
- this.dataProperties = list;
- }
- else {
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
- list.add(new CrossBlockOuterFirstDescriptor());
- list.add(new CrossBlockOuterSecondDescriptor());
- list.add(new CrossStreamOuterFirstDescriptor());
- list.add(new CrossStreamOuterSecondDescriptor());
- this.dataProperties = list;
- }
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public CrossOperatorBase<?, ?, ?, ?> getOperator() {
- return (CrossOperatorBase<?, ?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Cross";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return this.dataProperties;
- }
-
- /**
- * We assume that the cardinality is the product of the input cardinalities
- * and that the result width is the sum of the input widths.
- *
- * @param statistics The statistics object to optionally access.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
- long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
- this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 * card2;
-
- if (this.estimatedNumRecords >= 0) {
- float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
-
- if (width > 0) {
- this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
deleted file mode 100644
index 4e65976..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.plandump.DumpableConnection;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-/**
- * A connection between to operators. Represents an intermediate result
- * and a data exchange between the two operators.
- *
- * The data exchange has a mode in which it performs (batch / pipelined).
- *
- * The data exchange strategy may be set on this connection, in which case
- * it is fixed and will not be determined during candidate plan enumeration.
- *
- * During the enumeration of interesting properties, this connection also holds
- * all interesting properties generated by the successor operator.
- */
-public class DagConnection implements EstimateProvider, DumpableConnection<OptimizerNode> {
-
- private final OptimizerNode source; // The source node of the connection
-
- private final OptimizerNode target; // The target node of the connection.
-
- private final ExecutionMode dataExchangeMode; // defines whether to use batch or pipelined data exchange
-
- private InterestingProperties interestingProps; // local properties that succeeding nodes are interested in
-
- private ShipStrategyType shipStrategy; // The data shipping strategy, if predefined.
-
- private TempMode materializationMode = TempMode.NONE; // the materialization mode
-
- private int maxDepth = -1;
-
- private boolean breakPipeline; // whether this connection should break the pipeline due to potential deadlocks
-
- /**
- * Creates a new Connection between two nodes. The shipping strategy is by default <tt>NONE</tt>.
- * The temp mode is by default <tt>NONE</tt>.
- *
- * @param source
- * The source node.
- * @param target
- * The target node.
- */
- public DagConnection(OptimizerNode source, OptimizerNode target, ExecutionMode exchangeMode) {
- this(source, target, null, exchangeMode);
- }
-
- /**
- * Creates a new Connection between two nodes.
- *
- * @param source
- * The source node.
- * @param target
- * The target node.
- * @param shipStrategy
- * The shipping strategy.
- * @param exchangeMode
- * The data exchange mode (pipelined / batch / batch only for shuffles / ... )
- */
- public DagConnection(OptimizerNode source, OptimizerNode target,
- ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
- {
- if (source == null || target == null) {
- throw new NullPointerException("Source and target must not be null.");
- }
- this.source = source;
- this.target = target;
- this.shipStrategy = shipStrategy;
- this.dataExchangeMode = exchangeMode;
- }
-
- /**
- * Constructor to create a result from an operator that is not
- * consumed by another operator.
- *
- * @param source
- * The source node.
- */
- public DagConnection(OptimizerNode source, ExecutionMode exchangeMode) {
- if (source == null) {
- throw new NullPointerException("Source and target must not be null.");
- }
- this.source = source;
- this.target = null;
- this.shipStrategy = ShipStrategyType.NONE;
- this.dataExchangeMode = exchangeMode;
- }
-
- /**
- * Gets the source of the connection.
- *
- * @return The source Node.
- */
- public OptimizerNode getSource() {
- return this.source;
- }
-
- /**
- * Gets the target of the connection.
- *
- * @return The target node.
- */
- public OptimizerNode getTarget() {
- return this.target;
- }
-
- /**
- * Gets the shipping strategy for this connection.
- *
- * @return The connection's shipping strategy.
- */
- public ShipStrategyType getShipStrategy() {
- return this.shipStrategy;
- }
-
- /**
- * Sets the shipping strategy for this connection.
- *
- * @param strategy
- * The shipping strategy to be applied to this connection.
- */
- public void setShipStrategy(ShipStrategyType strategy) {
- this.shipStrategy = strategy;
- }
-
- /**
- * Gets the data exchange mode to use for this connection.
- *
- * @return The data exchange mode to use for this connection.
- */
- public ExecutionMode getDataExchangeMode() {
- if (dataExchangeMode == null) {
- throw new IllegalStateException("This connection does not have the data exchange mode set");
- }
- return dataExchangeMode;
- }
-
- /**
- * Marks that this connection should do a decoupled data exchange (such as batched)
- * rather then pipeline data. Connections are marked as pipeline breakers to avoid
- * deadlock situations.
- */
- public void markBreaksPipeline() {
- this.breakPipeline = true;
- }
-
- /**
- * Checks whether this connection is marked to break the pipeline.
- *
- * @return True, if this connection is marked to break the pipeline, false otherwise.
- */
- public boolean isBreakingPipeline() {
- return this.breakPipeline;
- }
-
- /**
- * Gets the interesting properties object for this pact connection.
- * If the interesting properties for this connections have not yet been set,
- * this method returns null.
- *
- * @return The collection of all interesting properties, or null, if they have not yet been set.
- */
- public InterestingProperties getInterestingProperties() {
- return this.interestingProps;
- }
-
- /**
- * Sets the interesting properties for this pact connection.
- *
- * @param props The interesting properties.
- */
- public void setInterestingProperties(InterestingProperties props) {
- if (this.interestingProps == null) {
- this.interestingProps = props;
- } else {
- throw new IllegalStateException("Interesting Properties have already been set.");
- }
- }
-
- public void clearInterestingProperties() {
- this.interestingProps = null;
- }
-
- public void initMaxDepth() {
-
- if (this.maxDepth == -1) {
- this.maxDepth = this.source.getMaxDepth() + 1;
- } else {
- throw new IllegalStateException("Maximum path depth has already been initialized.");
- }
- }
-
- public int getMaxDepth() {
- if (this.maxDepth != -1) {
- return this.maxDepth;
- } else {
- throw new IllegalStateException("Maximum path depth has not been initialized.");
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Estimates
- // --------------------------------------------------------------------------------------------
-
- @Override
- public long getEstimatedOutputSize() {
- return this.source.getEstimatedOutputSize();
- }
-
- @Override
- public long getEstimatedNumRecords() {
- return this.source.getEstimatedNumRecords();
- }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() {
- return this.source.getEstimatedAvgWidthPerOutputRecord();
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- public TempMode getMaterializationMode() {
- return this.materializationMode;
- }
-
- public void setMaterializationMode(TempMode materializationMode) {
- this.materializationMode = materializationMode;
- }
-
- public boolean isOnDynamicPath() {
- return this.source.isOnDynamicPath();
- }
-
- public int getCostWeight() {
- return this.source.getCostWeight();
- }
-
- // --------------------------------------------------------------------------------------------
-
- public String toString() {
- StringBuilder buf = new StringBuilder(50);
- buf.append("Connection: ");
-
- if (this.source == null) {
- buf.append("null");
- } else {
- buf.append(this.source.getOperator().getName());
- buf.append('(').append(this.source.getName()).append(')');
- }
-
- buf.append(" -> ");
-
- if (this.shipStrategy != null) {
- buf.append('[');
- buf.append(this.shipStrategy.name());
- buf.append(']').append(' ');
- }
-
- if (this.target == null) {
- buf.append("null");
- } else {
- buf.append(this.target.getOperator().getName());
- buf.append('(').append(this.target.getName()).append(')');
- }
-
- return buf.toString();
- }
-}
[27/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
deleted file mode 100644
index 8c19462..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
+++ /dev/null
@@ -1,1436 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import static org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport.*;
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.FlatJoinFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.operators.BinaryOperatorInformation;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.OperatorInformation;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.UnaryOperatorInformation;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dag.MapNode;
-import org.apache.flink.optimizer.dag.JoinNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.junit.Test;
-import org.apache.flink.api.java.io.TextInputFormat;
-
-
-public class FeedbackPropertiesMatchTest {
-
- @Test
- public void testNoPartialSolutionFoundSingleInputOnly() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
-
- SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
-
- Channel toMap1 = new Channel(target);
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(map1);
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = new LocalProperties();
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
- assertTrue(report == NO_PARTIAL_SOLUTION);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSingleInputOperators() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
-
- Channel toMap1 = new Channel(target);
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(map1);
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- // no feedback properties and none are ever required and present
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = new LocalProperties();
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some global feedback properties and none are ever required and present
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = new LocalProperties();
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some local feedback properties and none are ever required and present
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some global and local feedback properties and none are ever required and present
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // --------------------------- requirements on channel 1 -----------------------
-
- // some required global properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = new LocalProperties();
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldList(2, 5));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required local properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1, 2));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global and local properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldList(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1, 2));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global and local properties, which are over-fulfilled
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldSet(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global properties that are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 1));
- LocalProperties lp = new LocalProperties();
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldList(2, 5));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required local properties that are not met
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required global and local properties where the global properties are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 1));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required global and local properties where the local properties are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // --------------------------- requirements on channel 2 -----------------------
-
- // some required global properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = new LocalProperties();
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldList(2, 5));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required local properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1, 2));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global and local properties, which are matched exactly
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 5));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldList(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1, 2));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global and local properties, which are over-fulfilled
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldSet(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global properties that are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 1));
- LocalProperties lp = new LocalProperties();
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setHashPartitioned(new FieldSet(2, 5));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required local properties that are not met
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required global and local properties where the global properties are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(2, 1));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldSet(2, 5));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some required global and local properties where the local properties are not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // ---------------------- requirements mixed on 1 and 2 -----------------------
-
- // some required global properties at step one and some more at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.EMPTY;
-
- RequestedGlobalProperties reqGp1 = new RequestedGlobalProperties();
- reqGp1.setAnyPartitioning(new FieldList(1, 2));
-
- RequestedGlobalProperties reqGp2 = new RequestedGlobalProperties();
- reqGp2.setHashPartitioned(new FieldList(1, 2));
-
- toMap1.setRequiredGlobalProps(reqGp1);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp2);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required local properties at step one and some more at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
-
- RequestedLocalProperties reqLp1 = new RequestedLocalProperties();
- reqLp1.setGroupedFields(new FieldList(3, 1));
-
- RequestedLocalProperties reqLp2 = new RequestedLocalProperties();
- reqLp2.setOrdering(new Ordering(3, null, Order.ANY).appendOrdering(1, null, Order.ANY));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp1);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp2);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required global properties at step one and some local ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some required local properties at step one and some global ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some fulfilled global properties at step one and some non-fulfilled local ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 3));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some fulfilled local properties at step one and some non-fulfilled global ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(2, 3));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some non-fulfilled global properties at step one and some fulfilled local ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(2, 3));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // some non-fulfilled local properties at step one and some fulfilled global ones at step 2
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldList(1, 2));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(2, 1, 3));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSingleInputOperatorsWithReCreation() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
-
- Channel toMap1 = new Channel(target);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(map1);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- // set ship strategy in first channel, so later non matching global properties do not matter
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.EMPTY;
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldSet(2, 5));
-
- toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(reqGp);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(MET, report);
- }
-
- // set ship strategy in second channel, so previous non matching global properties void the match
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.EMPTY;
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldSet(2, 5));
-
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
-
- toMap2.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // set local strategy in first channel, so later non matching local properties do not matter
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(4, 1));
-
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // set local strategy in second channel, so previous non matching local properties void the match
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(4, 1));
-
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
-
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // create the properties on the same node as the requirement
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(1, 2));
- LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldSet(5, 7));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(5, 7));
-
- toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(5, 7), DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
- toMap1.setRequiredGlobalProps(reqGp);
- toMap1.setRequiredLocalProps(reqLp);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(MET, report);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testSingleInputOperatorsChainOfThree() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
-
- Channel toMap1 = new Channel(target);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(map1);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- Channel toMap3 = new Channel(map2);
- SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP);
-
- // set local strategy in first channel, so later non matching local properties do not matter
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
-
- RequestedLocalProperties reqLp = new RequestedLocalProperties();
- reqLp.setGroupedFields(new FieldList(4, 1));
-
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
- toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap3.setLocalStrategy(LocalStrategy.NONE);
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- toMap3.setRequiredGlobalProps(null);
- toMap3.setRequiredLocalProps(reqLp);
-
- FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // set global strategy in first channel, so later non matching global properties do not matter
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(5, 3));
- LocalProperties lp = LocalProperties.EMPTY;
-
- RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
- reqGp.setAnyPartitioning(new FieldSet(2, 3));
-
- toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(1, 2), DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
-
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
-
- toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap3.setLocalStrategy(LocalStrategy.NONE);
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toMap2.setRequiredGlobalProps(null);
- toMap2.setRequiredLocalProps(null);
-
- toMap3.setRequiredGlobalProps(reqGp);
- toMap3.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testNoPartialSolutionFoundTwoInputOperator() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
-
- SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
- SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
-
- Channel toMap1 = new Channel(source1);
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(source2);
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- Channel toJoin1 = new Channel(map1);
- Channel toJoin2 = new Channel(map2);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin2.setLocalStrategy(LocalStrategy.NONE);
-
- DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
- assertEquals(NO_PARTIAL_SOLUTION, report);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTwoOperatorsOneIndependent() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
- SourcePlanNode source = new SourcePlanNode(getSourceNode(), "Other Source");
-
- Channel toMap1 = new Channel(target);
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(source);
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- Channel toJoin1 = new Channel(map1);
- Channel toJoin2 = new Channel(map2);
-
- DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
-
- Channel toAfterJoin = new Channel(join);
- toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
-
- // attach some properties to the non-relevant input
- {
- toMap2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] {true, true});
-
- RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
- joinGp.setFullyReplicated();
-
- RequestedLocalProperties joinLp = new RequestedLocalProperties();
- joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
-
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin2.setLocalStrategy(LocalStrategy.NONE);
- toJoin2.setRequiredGlobalProps(joinGp);
- toJoin2.setRequiredLocalProps(joinLp);
- }
-
- // ------------------------------------------------------------------------------------
-
- // no properties from the partial solution, no required properties
- {
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.EMPTY;
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some properties from the partial solution, no required properties
- {
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
-
- // produced properties match relevant input
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2));
-
- toJoin1.setRequiredGlobalProps(rgp);
- toJoin1.setRequiredLocalProps(rlp);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // produced properties do not match relevant input
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(1, 2, 3));
-
- toJoin1.setRequiredGlobalProps(rgp);
- toJoin1.setRequiredLocalProps(rlp);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // produced properties overridden before join
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(rgp);
- toMap1.setRequiredLocalProps(rlp);
-
- toJoin1.setRequiredGlobalProps(null);
- toJoin1.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(MET, report);
- }
-
- // produced properties before join match, after join match as well
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2, 1));
-
- toMap1.setRequiredGlobalProps(null);
- toMap1.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- toJoin1.setRequiredGlobalProps(rgp);
- toJoin1.setRequiredLocalProps(rlp);
-
- toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // produced properties before join match, after join do not match
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
- rgp1.setHashPartitioned(new FieldList(0));
-
- RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
- rgp2.setHashPartitioned(new FieldList(3));
-
- RequestedLocalProperties rlp1 = new RequestedLocalProperties();
- rlp1.setGroupedFields(new FieldList(2, 1));
-
- RequestedLocalProperties rlp2 = new RequestedLocalProperties();
- rlp2.setGroupedFields(new FieldList(3, 4));
-
- toJoin1.setRequiredGlobalProps(rgp1);
- toJoin1.setRequiredLocalProps(rlp1);
-
- toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
-
- toAfterJoin.setRequiredGlobalProps(rgp2);
- toAfterJoin.setRequiredLocalProps(rlp2);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // produced properties are overridden, does not matter that they do not match
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setAnyPartitioning(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(1));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(1, 2, 3));
-
- toJoin1.setRequiredGlobalProps(null);
- toJoin1.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(MET, report);
- }
-
- // local property overridden before join, local property mismatch after join not relevant
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setAnyPartitioning(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(1, 2, 3));
-
- toJoin1.setRequiredGlobalProps(null);
- toJoin1.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
-
- toAfterJoin.setRequiredGlobalProps(null);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // local property overridden before join, global property mismatch after join void the match
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setAnyPartitioning(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(1));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(1, 2, 3));
-
- toJoin1.setRequiredGlobalProps(null);
- toJoin1.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTwoOperatorsBothDependent() {
- try {
- SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
-
- Channel toMap1 = new Channel(target);
- toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap1.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
-
- Channel toMap2 = new Channel(target);
- toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toMap2.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
-
- Channel toJoin1 = new Channel(map1);
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- Channel toJoin2 = new Channel(map2);
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin2.setLocalStrategy(LocalStrategy.NONE);
-
- DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
-
- Channel toAfterJoin = new Channel(join);
- toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
- SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
-
- // no properties from the partial solution, no required properties
- {
- GlobalProperties gp = new GlobalProperties();
- LocalProperties lp = LocalProperties.EMPTY;
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // some properties from the partial solution, no required properties
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // test requirements on one input and met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2, 1));
-
- toJoin1.setRequiredGlobalProps(rgp);
- toJoin1.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // test requirements on both input and met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2, 1));
-
- toJoin1.setRequiredGlobalProps(rgp);
- toJoin1.setRequiredLocalProps(rlp);
-
- toJoin2.setRequiredGlobalProps(rgp);
- toJoin2.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
- }
-
- // test requirements on both inputs, one not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
- rgp1.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp1 = new RequestedLocalProperties();
- rlp1.setGroupedFields(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
- rgp2.setHashPartitioned(new FieldList(1));
-
- RequestedLocalProperties rlp2 = new RequestedLocalProperties();
- rlp2.setGroupedFields(new FieldList(0, 3));
-
- toJoin1.setRequiredGlobalProps(rgp1);
- toJoin1.setRequiredLocalProps(rlp1);
-
- toJoin2.setRequiredGlobalProps(rgp2);
- toJoin2.setRequiredLocalProps(rlp2);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // test override on both inputs, later requirement ignored
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(1));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(0, 3));
-
- toJoin1.setRequiredGlobalProps(null);
- toJoin1.setRequiredLocalProps(null);
-
- toJoin2.setRequiredGlobalProps(null);
- toJoin2.setRequiredLocalProps(null);
-
- toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
- toJoin2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(MET, report);
- }
-
- // test override on one inputs, later requirement met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(0));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(2, 1));
-
- toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(PENDING, report);
- }
-
- // test override on one input, later requirement not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(3));
-
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setGroupedFields(new FieldList(77, 69));
-
- toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(rlp);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
-
- // test override on one input locally, later global requirement not met
- {
- GlobalProperties gp = new GlobalProperties();
- gp.setHashPartitioned(new FieldList(0));
- LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
-
- RequestedGlobalProperties rgp = new RequestedGlobalProperties();
- rgp.setHashPartitioned(new FieldList(3));
-
-
- toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(3), new boolean[] { false });
-
- toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
- toJoin1.setLocalStrategy(LocalStrategy.NONE);
-
- toAfterJoin.setRequiredGlobalProps(rgp);
- toAfterJoin.setRequiredLocalProps(null);
-
- FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
- assertEquals(NOT_MET, report);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static DataSourceNode getSourceNode() {
- return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>(new TextInputFormat(new Path("/")), new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO)));
- }
-
- private static MapNode getMapNode() {
- return new MapNode(new MapOperatorBase<String, String, MapFunction<String,String>>(new IdentityMapper<String>(), new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), "map op"));
- }
-
- private static JoinNode getJoinNode() {
- return new JoinNode(new JoinOperatorBase<String, String, String, FlatJoinFunction<String, String, String>>(new DummyFlatJoinFunction<String>(), new BinaryOperatorInformation<String, String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[] {1}, new int[] {2}, "join op"));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
deleted file mode 100644
index 77d185d..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.record.operators.CoGroupOperator;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.util.DummyCoGroupStub;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.types.DoubleValue;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.StringValue;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * This test case has been created to validate that correct strategies are used if orders within groups are
- * requested.
- */
-@SuppressWarnings({"serial", "deprecation"})
-public class GroupOrderTest extends CompilerTestBase {
-
- @Test
- public void testReduceWithGroupOrder() {
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
-
- ReduceOperator reduce = ReduceOperator.builder(new IdentityReduce()).keyField(IntValue.class, 2).name("Reduce").input(source).build();
- Ordering groupOrder = new Ordering(5, StringValue.class, Order.DESCENDING);
- reduce.setGroupOrder(groupOrder);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, reduce, "Sink");
-
-
- Plan plan = new Plan(sink, "Test Temp Task");
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
- OptimizedPlan oPlan;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- return; // silence the compiler
- }
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- SinkPlanNode sinkNode = resolver.getNode("Sink");
- SingleInputPlanNode reducer = resolver.getNode("Reduce");
-
- // verify the strategies
- Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
- Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
-
- Channel c = reducer.getInput();
- Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
-
- FieldList ship = new FieldList(2);
- FieldList local = new FieldList(2, 5);
- Assert.assertEquals(ship, c.getShipStrategyKeys());
- Assert.assertEquals(local, c.getLocalStrategyKeys());
- Assert.assertTrue(c.getLocalStrategySortOrder()[0] == reducer.getSortOrders(0)[0]);
-
- // check that we indeed sort descending
- Assert.assertTrue(c.getLocalStrategySortOrder()[1] == groupOrder.getFieldSortDirections()[0]);
- }
-
- @Test
- public void testCoGroupWithGroupOrder() {
- // construct the plan
- FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source1");
- FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source2");
-
- CoGroupOperator coGroup = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 3, 6)
- .keyField(LongValue.class, 0, 0)
- .name("CoGroup").input1(source1).input2(source2).build();
-
- Ordering groupOrder1 = new Ordering(5, StringValue.class, Order.DESCENDING);
- Ordering groupOrder2 = new Ordering(1, StringValue.class, Order.DESCENDING);
- groupOrder2.appendOrdering(4, DoubleValue.class, Order.ASCENDING);
- coGroup.setGroupOrderForInputOne(groupOrder1);
- coGroup.setGroupOrderForInputTwo(groupOrder2);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, coGroup, "Sink");
-
- Plan plan = new Plan(sink, "Reduce Group Order Test");
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
- OptimizedPlan oPlan;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- return; // silence the compiler
- }
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- SinkPlanNode sinkNode = resolver.getNode("Sink");
- DualInputPlanNode coGroupNode = resolver.getNode("CoGroup");
-
- // verify the strategies
- Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
- Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy());
- Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy());
-
- Channel c1 = coGroupNode.getInput1();
- Channel c2 = coGroupNode.getInput2();
-
- Assert.assertEquals(LocalStrategy.SORT, c1.getLocalStrategy());
- Assert.assertEquals(LocalStrategy.SORT, c2.getLocalStrategy());
-
- FieldList ship1 = new FieldList(new int[] {3, 0});
- FieldList ship2 = new FieldList(new int[] {6, 0});
-
- FieldList local1 = new FieldList(new int[] {3, 0, 5});
- FieldList local2 = new FieldList(new int[] {6, 0, 1, 4});
-
- Assert.assertEquals(ship1, c1.getShipStrategyKeys());
- Assert.assertEquals(ship2, c2.getShipStrategyKeys());
- Assert.assertEquals(local1, c1.getLocalStrategyKeys());
- Assert.assertEquals(local2, c2.getLocalStrategyKeys());
-
- Assert.assertTrue(c1.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
- Assert.assertTrue(c1.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
- Assert.assertTrue(c2.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
- Assert.assertTrue(c2.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
-
- // check that the local group orderings are correct
- Assert.assertTrue(c1.getLocalStrategySortOrder()[2] == groupOrder1.getFieldSortDirections()[0]);
- Assert.assertTrue(c2.getLocalStrategySortOrder()[2] == groupOrder2.getFieldSortDirections()[0]);
- Assert.assertTrue(c2.getLocalStrategySortOrder()[3] == groupOrder2.getFieldSortDirections()[1]);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
deleted file mode 100644
index 6dadc19..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.record.operators.CrossOperator;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.DummyCrossStub;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityMap;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.apache.flink.types.IntValue;
-import org.junit.Test;
-
-/**
- * This class tests plans that once failed because of a bug:
- * <ul>
- * <li> Ticket 158
- * </ul>
- */
-@SuppressWarnings({"serial", "deprecation"})
-public class HardPlansCompilationTest extends CompilerTestBase {
-
- /**
- * Source -> Map -> Reduce -> Cross -> Reduce -> Cross -> Reduce ->
- * |--------------------------/ /
- * |--------------------------------------------/
- *
- * First cross has SameKeyFirst output contract
- */
- @Test
- public void testTicket158() {
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
-
- MapOperator map = MapOperator.builder(new IdentityMap()).name("Map1").input(source).build();
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce1").input(map).build();
-
- CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub()).name("Cross1").input1(reduce1).input2(source).build();
-
- ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce2").input(cross1).build();
-
- CrossOperator cross2 = CrossOperator.builder(new DummyCrossStub()).name("Cross2").input1(reduce2).input2(source).build();
-
- ReduceOperator reduce3 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce3").input(cross2).build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setInput(reduce3);
-
- Plan plan = new Plan(sink, "Test Temp Task");
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
- OptimizedPlan oPlan = compileNoStats(plan);
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
deleted file mode 100644
index ac4f820..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichJoinFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Collector;
-
-
-@SuppressWarnings({"serial", "unchecked"})
-public class IterationsCompilerTest extends CompilerTestBase {
-
- @Test
- public void testSolutionSetDeltaDependsOnBroadcastVariable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> source =
- env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
-
- DataSet<Tuple2<Long, Long>> invariantInput =
- env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
-
- // iteration from here
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = source.iterateDelta(source, 1000, 1);
-
- DataSet<Tuple2<Long, Long>> result =
- invariantInput
- .map(new IdentityMapper<Tuple2<Long, Long>>()).withBroadcastSet(iter.getWorkset(), "bc data")
- .join(iter.getSolutionSet()).where(0).equalTo(1).projectFirst(1).projectSecond(1);
-
- iter.closeWith(result.map(new IdentityMapper<Tuple2<Long,Long>>()), result).print();
-
- OptimizedPlan p = compileNoStats(env.createProgramPlan());
-
- // check that the JSON generator accepts this plan
- new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(p);
-
- // check that the JobGraphGenerator accepts the plan
- new JobGraphGenerator().compileJobGraph(p);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTwoIterationsWithMapperInbetween() throws Exception {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> bulkResult = doBulkIteration(verticesWithInitialId, edges);
-
- DataSet<Tuple2<Long, Long>> mappedBulk = bulkResult.map(new DummyMap());
-
- DataSet<Tuple2<Long, Long>> depResult = doDeltaIteration(mappedBulk, edges);
-
- depResult.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(1, op.getDataSinks().size());
- assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
-
- WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
- assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTwoIterationsDirectlyChained() throws Exception {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> bulkResult = doBulkIteration(verticesWithInitialId, edges);
-
- DataSet<Tuple2<Long, Long>> depResult = doDeltaIteration(bulkResult, edges);
-
- depResult.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(1, op.getDataSinks().size());
- assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
-
- WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
- assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testTwoWorksetIterationsDirectlyChained() throws Exception {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
-
- DataSet<Tuple2<Long, Long>> firstResult = doDeltaIteration(verticesWithInitialId, edges);
-
- DataSet<Tuple2<Long, Long>> secondResult = doDeltaIteration(firstResult, edges);
-
- secondResult.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(1, op.getDataSinks().size());
- assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
-
- WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy());
- assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testIterationPushingWorkOut() throws Exception {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<Long, Long>> input1 = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
-
- DataSet<Tuple2<Long, Long>> input2 = env.readCsvFile("/some/file/path").types(Long.class, Long.class);
-
- doBulkIteration(input1, input2).print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(1, op.getDataSinks().size());
- assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof BulkIterationPlanNode);
-
- BulkIterationPlanNode bipn = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- // check that work has not! been pushed out, as the end of the step function does not produce the necessary properties
- for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) {
- assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy());
- }
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testWorksetIterationPipelineBreakerPlacement() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- // the workset (input two of the delta iteration) is the same as what is consumed be the successive join
- DataSet<Tuple2<Long, Long>> initialWorkset = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
-
- DataSet<Tuple2<Long, Long>> initialSolutionSet = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
-
- // trivial iteration, since we are interested in the inputs to the iteration
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = initialSolutionSet.iterateDelta(initialWorkset, 100, 0);
-
- DataSet<Tuple2<Long, Long>> next = iteration.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>());
-
- DataSet<Tuple2<Long, Long>> result = iteration.closeWith(next, next);
-
- initialWorkset
- .join(result, JoinHint.REPARTITION_HASH_FIRST)
- .where(0).equalTo(0)
- .print();
-
- Plan p = env.createProgramPlan();
- compileNoStats(p);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testResetPartialSolution() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> width = env.generateSequence(1, 10);
- DataSet<Long> update = env.generateSequence(1, 10);
- DataSet<Long> lastGradient = env.generateSequence(1, 10);
-
- DataSet<Long> init = width.union(update).union(lastGradient);
-
- IterativeDataSet<Long> iteration = init.iterate(10);
-
- width = iteration.filter(new IdFilter<Long>());
- update = iteration.filter(new IdFilter<Long>());
- lastGradient = iteration.filter(new IdFilter<Long>());
-
- DataSet<Long> gradient = width.map(new IdentityMapper<Long>());
- DataSet<Long> term = gradient.join(lastGradient)
- .where(new IdentityKeyExtractor<Long>())
- .equalTo(new IdentityKeyExtractor<Long>())
- .with(new JoinFunction<Long, Long, Long>() {
- public Long join(Long first, Long second) { return null; }
- });
-
- update = update.map(new RichMapFunction<Long, Long>() {
- public Long map(Long value) { return null; }
- }).withBroadcastSet(term, "some-name");
-
- DataSet<Long> result = iteration.closeWith(width.union(update).union(lastGradient));
-
- result.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static DataSet<Tuple2<Long, Long>> doBulkIteration(DataSet<Tuple2<Long, Long>> vertices, DataSet<Tuple2<Long, Long>> edges) {
-
- // open a bulk iteration
- IterativeDataSet<Tuple2<Long, Long>> iteration = vertices.iterate(20);
-
- DataSet<Tuple2<Long, Long>> changes = iteration
- .join(edges).where(0).equalTo(0).with(new Join222())
- .groupBy(0).aggregate(Aggregations.MIN, 1)
- .join(iteration).where(0).equalTo(0)
- .flatMap(new FlatMapJoin());
-
- // close the bulk iteration
- return iteration.closeWith(changes);
- }
-
-
- public static DataSet<Tuple2<Long, Long>> doDeltaIteration(DataSet<Tuple2<Long, Long>> vertices, DataSet<Tuple2<Long, Long>> edges) {
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> depIteration = vertices.iterateDelta(vertices, 100, 0);
-
- DataSet<Tuple1<Long>> candidates = depIteration.getWorkset().join(edges).where(0).equalTo(0)
- .projectSecond(1);
-
- DataSet<Tuple1<Long>> grouped = candidates.groupBy(0).reduceGroup(new Reduce101());
-
- DataSet<Tuple2<Long, Long>> candidatesDependencies =
- grouped.join(edges).where(0).equalTo(1).projectSecond(0, 1);
-
- DataSet<Tuple2<Long, Long>> verticesWithNewComponents =
- candidatesDependencies.join(depIteration.getSolutionSet()).where(0).equalTo(0)
- .with(new Join222())
- .groupBy(0).aggregate(Aggregations.MIN, 1);
-
- DataSet<Tuple2<Long, Long>> updatedComponentId =
- verticesWithNewComponents.join(depIteration.getSolutionSet()).where(0).equalTo(0)
- .flatMap(new FlatMapJoin());
-
- DataSet<Tuple2<Long, Long>> depResult = depIteration.closeWith(updatedComponentId, updatedComponentId);
-
- return depResult;
-
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static final class Join222 extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- @Override
- public Tuple2<Long, Long> join(Tuple2<Long, Long> vertexWithComponent, Tuple2<Long, Long> edge) {
- return null;
- }
- }
-
- public static final class FlatMapJoin extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
-
- @Override
- public void flatMap(Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> value, Collector<Tuple2<Long, Long>> out) {}
- }
-
- public static final class DummyMap extends RichMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
- @Override
- public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception {
- return value;
- }
- }
-
- @ForwardedFields("0")
- public static final class Reduce101 extends RichGroupReduceFunction<Tuple1<Long>, Tuple1<Long>> {
-
- @Override
- public void reduce(Iterable<Tuple1<Long>> values, Collector<Tuple1<Long>> out) {}
- }
-
- @ForwardedFields("0")
- public static final class DuplicateValue extends RichMapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
-
- @Override
- public Tuple2<Long, Long> map(Tuple1<Long> value) throws Exception {
- return new Tuple2<Long, Long>(value.f0, value.f0);
- }
- }
-
- public static final class DuplicateValueScalar<T> extends RichMapFunction<T, Tuple2<T, T>> {
-
- @Override
- public Tuple2<T, T> map(T value) {
- return new Tuple2<T, T>(value, value);
- }
- }
-
- public static final class IdFilter<T> implements FilterFunction<T> {
- @Override
- public boolean filter(T value) {
- return true;
- }
- }
-}
[50/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
index ced0e83..6946641 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.dataproperties;
import java.util.HashSet;
@@ -29,13 +28,16 @@ import org.apache.flink.optimizer.dag.SingleInputNode;
import org.apache.flink.optimizer.dag.TwoInputNode;
/**
- * The interesting properties that a node in the optimizer plan hands to its predecessors. It has the
- * purpose to tell the preceding nodes, which data properties might have the advantage, because they would
- * let the node fulfill its pact cheaper. More on optimization with interesting properties can be found
- * in the works on the volcano- and cascades optimizer framework.
+ * Interesting properties are propagated from parent operators to child operators. They tell the child
+ * what data properties would help the parent in operating in a cheaper fashion. A reduce operator, for
+ * example, tells its child that partitioned data would help. If the child is a join operator, it can use
+ * that knowledge to favor strategies that leave the data in a partitioned form.
+ *
+ * More on optimization with interesting properties can be found in the works on
+ * the volcano- and cascades optimizer framework.
*/
-public class InterestingProperties implements Cloneable
-{
+public class InterestingProperties implements Cloneable {
+
private Set<RequestedGlobalProperties> globalProps; // the global properties, i.e. properties across partitions
private Set<RequestedLocalProperties> localProps; // the local properties, i.e. properties within partitions
@@ -91,8 +93,7 @@ public class InterestingProperties implements Cloneable
return this.globalProps;
}
- public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input)
- {
+ public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input) {
InterestingProperties iProps = new InterestingProperties();
SemanticProperties props;
if (node instanceof SingleInputNode || node instanceof TwoInputNode) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
index 0c3ea12..e0231aa 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
/**
* This class represents local properties of the data. A local property is a property that exists
- * within the data of a single partition.
+ * within the data of a single partition, such as sort order, or data grouping.
*/
public class LocalProperties implements Cloneable {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
index 674cdb8..5e06dd3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
@@ -19,42 +19,45 @@
package org.apache.flink.optimizer.dataproperties;
/**
- * An enumeration tracking the different types of sharding strategies.
+ * An enumeration of the the different types of distributing data across partitions or
+ * parallel workers.
*/
public enum PartitioningProperty {
/**
- * Any data distribution, i.e., random partitioning or full replication.
+ * Any possible way of data distribution, including random partitioning and full replication.
*/
ANY_DISTRIBUTION,
/**
- * Constant indicating no particular partitioning (i.e. random) data distribution.
+ * A random disjunct (non-replicated) data distribution, where each datum is contained in one partition only.
+ * This is for example the result of parallel scans of data in a file system like HDFS,
+ * or the result of a round-robin data distribution.
*/
RANDOM_PARTITIONED,
/**
- * Constant indicating a hash partitioning.
+ * A hash partitioning on a certain key.
*/
HASH_PARTITIONED,
/**
- * Constant indicating a range partitioning.
+ * A range partitioning on a certain key.
*/
RANGE_PARTITIONED,
/**
- * Constant indicating any not further specified disjunct partitioning.
+ * A not further specified partitioning on a key (hash-, or range partitioning, or some other scheme even).
*/
ANY_PARTITIONING,
/**
- * Constant indicating full replication of the data to each parallel instance.
+ *Full replication of the data to each parallel instance.
*/
FULL_REPLICATION,
/**
- * Constant indicating a forced even re-balancing.
+ * A forced even re-balancing. All partitions are guaranteed to have almost the same number of records.
*/
FORCED_REBALANCED,
@@ -95,10 +98,10 @@ public enum PartitioningProperty {
/**
* Checks, if this property represents a partitioning that is computable.
- * Computable partitionings can be recreated through an algorithm. If two sets of data are to
+ * A computable partitioning can be recreated through an algorithm. If two sets of data are to
* be co-partitioned, it is crucial, that the partitioning schemes are computable.
* <p>
- * Examples for computable partitioning schemes are hash- or range-partitionings. An example for a non-computable
+ * Examples for computable partitioning schemes are hash- or range-partitioning. An example for a non-computable
* partitioning is the implicit partitioning that exists though a globally unique key.
*
* @return True, if this enum constant is a re-computable partitioning.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
index 102a0f0..a9d14e8 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
@@ -29,14 +29,16 @@ import org.apache.flink.optimizer.util.Utils;
import org.apache.flink.runtime.operators.util.LocalStrategy;
/**
- * This class represents local properties of the data. A local property is a property that exists
- * within the data of a single partition.
+ * This class represents the local properties of the data that are requested by an operator.
+ * Local properties are the properties within one partition.
+ * Operators request the local properties they need for correct execution. Here are some example local
+ * properties requested by certain operators:
+ * <ul>
+ * <li>"groupBy/reduce" will request the data to be grouped on the key fields.</li>
+ * <li>A sort-merge join will request the data from each input to be sorted on the respective join key.</li>
+ * </ul>
*/
public class RequestedLocalProperties implements Cloneable {
-
- public static final RequestedLocalProperties DEFAULT_PROPERTIES = null;
-
- // --------------------------------------------------------------------------------------------
private Ordering ordering; // order inside a partition, null if not ordered
@@ -205,9 +207,9 @@ public class RequestedLocalProperties implements Cloneable {
}
/**
- * Parameterizes the local strategy fields of a channel such that the channel produces the desired local properties.
+ * Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
*
- * @param channel The channel to parameterize.
+ * @param channel The channel to parametrize.
*/
public void parameterizeChannel(Channel channel) {
LocalProperties current = channel.getLocalProperties();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
index eaf11ac..9ee56b0 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
@@ -40,7 +40,7 @@ public final class AllGroupCombineProperties extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupCombine ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_COMBINE);
+ return new SingleInputPlanNode(node, "GroupCombine ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_COMBINE);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
index 55d6fbb..9efd8c7 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
@@ -40,7 +40,7 @@ public final class AllGroupReduceProperties extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
index 7d07e7d..b3c083a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupWithPartialPreGroupProperties.java
@@ -46,7 +46,7 @@ public final class AllGroupWithPartialPreGroupProperties extends OperatorDescrip
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
// locally connected, directly instantiate
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")",
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
in, DriverStrategy.ALL_GROUP_REDUCE);
} else {
// non forward case.plug in a combiner
@@ -55,10 +55,10 @@ public final class AllGroupWithPartialPreGroupProperties extends OperatorDescrip
// create an input node for combine with same DOP as input node
GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
+ "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
combiner.setCosts(new Costs(0, 0));
combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
@@ -67,7 +67,7 @@ public final class AllGroupWithPartialPreGroupProperties extends OperatorDescrip
in.getShipStrategySortOrder(), in.getDataExchangeMode());
toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")",
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
toReducer, DriverStrategy.ALL_GROUP_REDUCE);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
index 4f6a4fd..a172a60 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllReduceProperties.java
@@ -45,7 +45,7 @@ public final class AllReduceProperties extends OperatorDescriptorSingle {
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
// locally connected, directly instantiate
- return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")",
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
in, DriverStrategy.ALL_REDUCE);
} else {
// non forward case.plug in a combiner
@@ -54,10 +54,10 @@ public final class AllReduceProperties extends OperatorDescriptorSingle {
// create an input node for combine with same DOP as input node
ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
+ "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
combiner.setCosts(new Costs(0, 0));
combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
@@ -67,7 +67,7 @@ public final class AllReduceProperties extends OperatorDescriptorSingle {
toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
in.getLocalStrategySortOrder());
- return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")",
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
toReducer, DriverStrategy.ALL_REDUCE);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
index c46ce56..f48e297 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CartesianProductDescriptor.java
@@ -93,7 +93,7 @@ public abstract class CartesianProductDescriptor extends OperatorDescriptorDual
@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- return new DualInputPlanNode(node, "Cross("+node.getPactContract().getName()+")", in1, in2, getStrategy());
+ return new DualInputPlanNode(node, "Cross("+node.getOperator().getName()+")", in1, in2, getStrategy());
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
index a17063e..368944e 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
@@ -215,7 +215,7 @@ public class CoGroupDescriptor extends OperatorDescriptorDual {
inputOrders = tmp;
}
- return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
+ return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
index ffae66f..8e7edeb 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CoGroupWithSolutionSetFirstDescriptor.java
@@ -60,7 +60,7 @@ public class CoGroupWithSolutionSetFirstDescriptor extends CoGroupDescriptor {
inputOrders = tmp;
}
- return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
+ return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
index 92f7474..bcd4d73 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/CollectorMapDescriptor.java
@@ -42,7 +42,7 @@ public class CollectorMapDescriptor extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Map ("+node.getPactContract().getName()+")", in, DriverStrategy.COLLECTOR_MAP);
+ return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.COLLECTOR_MAP);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
index e7ff210..81c823f 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
@@ -41,7 +41,7 @@ public class FilterDescriptor extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Filter ("+node.getPactContract().getName()+")", in, DriverStrategy.FLAT_MAP);
+ return new SingleInputPlanNode(node, "Filter ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
index 5384aa6..b915e45 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
@@ -42,7 +42,7 @@ public class FlatMapDescriptor extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "FlatMap ("+node.getPactContract().getName()+")", in, DriverStrategy.FLAT_MAP);
+ return new SingleInputPlanNode(node, "FlatMap ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
index d0e839a..b648386 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
@@ -71,12 +71,12 @@ public final class GroupCombineProperties extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- node.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ node.setDegreeOfParallelism(in.getSource().getParallelism());
// sorting key info
SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode(
node,
- "GroupCombine (" + node.getPactContract().getName() + ")",
+ "GroupCombine (" + node.getOperator().getName() + ")",
in, // reuse the combine strategy also used in the group reduce
DriverStrategy.SORTED_GROUP_COMBINE, this.keyList);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
index c66321d..ebd09f2 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
@@ -85,7 +85,7 @@ public final class GroupReduceProperties extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
index 1caee6c..c4f47d3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
@@ -98,7 +98,7 @@ public final class GroupReduceWithCombineProperties extends OperatorDescriptorSi
in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
in.getLocalStrategySortOrder());
}
- return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", in,
+ return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", in,
DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
} else {
// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
@@ -107,9 +107,9 @@ public final class GroupReduceWithCombineProperties extends OperatorDescriptorSi
// create an input node for combine with same DOP as input node
GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getPactContract()
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator()
.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
combiner.setCosts(new Costs(0, 0));
combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
@@ -124,7 +124,7 @@ public final class GroupReduceWithCombineProperties extends OperatorDescriptorSi
toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
in.getLocalStrategySortOrder());
- return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")",
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
index a6c4500..fec72a9 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
@@ -79,7 +79,7 @@ public class HashJoinBuildFirstProperties extends AbstractJoinDescriptor {
else {
strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
}
- return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
+ return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
index 79cb3cb..f9d1e6c 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
@@ -78,7 +78,7 @@ public final class HashJoinBuildSecondProperties extends AbstractJoinDescriptor
else {
strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
}
- return new DualInputPlanNode(node, "Join ("+node.getPactContract().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
+ return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
index e55a728..9f14d2a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
@@ -41,7 +41,7 @@ public class MapDescriptor extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Map ("+node.getPactContract().getName()+")", in, DriverStrategy.MAP);
+ return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.MAP);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
index 8cef12d..1489097 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
@@ -41,7 +41,7 @@ public class MapPartitionDescriptor extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "MapPartition ("+node.getPactContract().getName()+")", in, DriverStrategy.MAP_PARTITION);
+ return new SingleInputPlanNode(node, "MapPartition ("+node.getOperator().getName()+")", in, DriverStrategy.MAP_PARTITION);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
index fe9302a..2bde29b 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
@@ -48,10 +48,10 @@ public final class PartialGroupProperties extends OperatorDescriptorSingle {
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
// create in input node for combine with same DOP as input node
- GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getPactContract());
- combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getPactContract().getName()+")", in,
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator().getName()+")", in,
DriverStrategy.SORTED_GROUP_COMBINE);
// sorting key info
combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
index 01096f0..5bb51f3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
@@ -61,7 +61,7 @@ public final class ReduceProperties extends OperatorDescriptorSingle {
if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
(node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty()))
{
- return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")", in,
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
DriverStrategy.SORTED_REDUCE, this.keyList);
}
else {
@@ -71,10 +71,10 @@ public final class ReduceProperties extends OperatorDescriptorSingle {
// create an input node for combine with same DOP as input node
ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getPactContract().getName()+")", toCombiner,
+ "Combine ("+node.getOperator().getName()+")", toCombiner,
DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList);
combiner.setCosts(new Costs(0, 0));
@@ -85,7 +85,7 @@ public final class ReduceProperties extends OperatorDescriptorSingle {
in.getShipStrategySortOrder(), in.getDataExchangeMode());
toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
- return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", toReducer,
+ return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", toReducer,
DriverStrategy.SORTED_REDUCE, this.keyList);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
index afe7e8d..356836a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
@@ -99,7 +99,7 @@ public class SortMergeJoinDescriptor extends AbstractJoinDescriptor {
inputOrders = tmp;
}
- return new DualInputPlanNode(node, "Join("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.MERGE, this.keys1, this.keys2, inputOrders);
+ return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, DriverStrategy.MERGE, this.keys1, this.keys2, inputOrders);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
index 093e968..bf22fb3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
@@ -42,7 +42,7 @@ public class BinaryUnionPlanNode extends DualInputPlanNode {
this.nodeCosts = toSwapFrom.nodeCosts;
this.cumulativeCosts = toSwapFrom.cumulativeCosts;
- setDegreeOfParallelism(toSwapFrom.getDegreeOfParallelism());
+ setParallelism(toSwapFrom.getParallelism());
}
public BinaryUnionNode getOptimizerNode() {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
index 8a2398c..875d1c3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
@@ -37,7 +37,30 @@ import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
/**
- * A Channel is a data exchange between two operators.
+ * A Channel represents the result produced by an operator and the data exchange
+ * before the consumption by the target operator.
+ *
+ * The channel defines and tracks various properties and characteristics of the
+ * data set and data exchange.
+ *
+ * Data set characteristics:
+ * <ul>
+ * <li>The "global properties" of the data, i.e., how the data is distributed across
+ * partitions</li>
+ * <li>The "required global properties" of the data, i.e., the global properties that, if absent,
+ * would cause the program to return a wrong result.</li>
+ * <li>The "local properties" of the data, i.e., how the data is organized within a partition</li>
+ * <li>The "required local properties" of the data, i.e., the local properties that, if absent,
+ * would cause the program to return a wrong result.</li>
+ * </ul>
+ *
+ * Data exchange parameters:
+ * <ul>
+ * <li>The "ship strategy", i.e., whether to forward the data, shuffle it, broadcast it, ...</li>
+ * <li>The "ship keys", which are the positions of the key fields in the exchanged records.</li>
+ * <li>The "data exchange mode", which defines whether to pipeline or batch the exchange</li>
+ * <li>Several more...</li>
+ * </ul>
*/
public class Channel implements EstimateProvider, Cloneable, DumpableConnection<PlanNode> {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
index eea9b67..01c56dd 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
@@ -81,10 +81,10 @@ public class DualInputPlanNode extends PlanNode {
this.sortOrders = driverSortOrders;
if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input1.setReplicationFactor(getDegreeOfParallelism());
+ this.input1.setReplicationFactor(getParallelism());
}
if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input2.setReplicationFactor(getDegreeOfParallelism());
+ this.input2.setReplicationFactor(getParallelism());
}
mergeBranchPlanMaps(input1.getSource(), input2.getSource());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
index e565909..6f634fb 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.plan;
import java.util.ArrayList;
@@ -73,7 +72,7 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
private double relativeMemoryPerSubTask; // the amount of memory dedicated to each task, in bytes
- private int degreeOfParallelism;
+ private int parallelism;
private boolean pFlag; // flag for the internal pruning algorithm
@@ -86,7 +85,7 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
this.nodeName = nodeName;
this.driverStrategy = strategy;
- this.degreeOfParallelism = template.getDegreeOfParallelism();
+ this.parallelism = template.getParallelism();
// check, if there is branch at this node. if yes, this candidate must be associated with
// the branching template node.
@@ -134,21 +133,21 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
// --------------------------------------------------------------------------------------------
/**
- * Gets the optimizer's pact node for which this plan candidate node was created.
+ * Gets the node from the optimizer DAG for which this plan candidate node was created.
*
- * @return The template optimizer's node.
+ * @return The optimizer's DAG node.
*/
public OptimizerNode getOriginalOptimizerNode() {
return this.template;
}
/**
- * Gets the pact contract this node represents in the plan.
+ * Gets the program operator that this node represents in the plan.
*
- * @return The pact contract this node represents in the plan.
+ * @return The program operator this node represents in the plan.
*/
- public Operator<?> getPactContract() {
- return this.template.getPactContract();
+ public Operator<?> getProgramOperator() {
+ return this.template.getOperator();
}
/**
@@ -252,7 +251,7 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
return null;
} else {
Costs result = cumulativeCosts.clone();
- if (this.template != null && this.template.getOutgoingConnections() != null) {
+ if (this.template.getOutgoingConnections() != null) {
int outDegree = this.template.getOutgoingConnections().size();
if (outDegree > 0) {
result.divideBy(outDegree);
@@ -302,12 +301,12 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
}
}
- public void setDegreeOfParallelism(int parallelism) {
- this.degreeOfParallelism = parallelism;
+ public void setParallelism(int parallelism) {
+ this.parallelism = parallelism;
}
- public int getDegreeOfParallelism() {
- return this.degreeOfParallelism;
+ public int getParallelism() {
+ return this.parallelism;
}
public long getGuaranteedAvailableMemory() {
@@ -514,7 +513,7 @@ public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<Plan
@Override
public String toString() {
- return this.template.getName() + " \"" + getPactContract().getName() + "\" : " + this.driverStrategy +
+ return this.template.getName() + " \"" + getProgramOperator().getName() + "\" : " + this.driverStrategy +
" [[ " + this.globalProps + " ]] [[ " + this.localProps + " ]]";
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
index cefd704..b928be7 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
@@ -81,7 +81,7 @@ public class SingleInputPlanNode extends PlanNode {
}
if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input.setReplicationFactor(getDegreeOfParallelism());
+ this.input.setReplicationFactor(getParallelism());
}
final PlanNode predNode = input.getSource();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
index 4083a2a..6f918c0 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
@@ -39,7 +39,7 @@ import org.apache.flink.optimizer.dag.BulkIterationNode;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.dag.DataSourceNode;
import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.PactConnection;
+import org.apache.flink.optimizer.dag.DagConnection;
import org.apache.flink.optimizer.dag.TempMode;
import org.apache.flink.optimizer.dag.WorksetIterationNode;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
@@ -231,18 +231,18 @@ public class PlanJSONDumpGenerator {
String contents;
if (n instanceof DataSinkNode) {
type = "sink";
- contents = n.getPactContract().toString();
+ contents = n.getOperator().toString();
} else if (n instanceof DataSourceNode) {
type = "source";
- contents = n.getPactContract().toString();
+ contents = n.getOperator().toString();
}
else if (n instanceof BulkIterationNode) {
type = "bulk_iteration";
- contents = n.getPactContract().getName();
+ contents = n.getOperator().getName();
}
else if (n instanceof WorksetIterationNode) {
type = "workset_iteration";
- contents = n.getPactContract().getName();
+ contents = n.getOperator().getName();
}
else if (n instanceof BinaryUnionNode) {
type = "pact";
@@ -250,7 +250,7 @@ public class PlanJSONDumpGenerator {
}
else {
type = "pact";
- contents = n.getPactContract().getName();
+ contents = n.getOperator().getName();
}
contents = StringUtils.showControlCharacters(contents);
@@ -277,7 +277,7 @@ public class PlanJSONDumpGenerator {
// degree of parallelism
writer.print(",\n\t\t\"parallelism\": \""
- + (n.getDegreeOfParallelism() >= 1 ? n.getDegreeOfParallelism() : "default") + "\"");
+ + (n.getParallelism() >= 1 ? n.getParallelism() : "default") + "\"");
// output node predecessors
Iterator<? extends DumpableConnection<?>> inConns = node.getDumpableInputs().iterator();
@@ -294,10 +294,10 @@ public class PlanJSONDumpGenerator {
writer.print(inputNum == 0 ? "\n" : ",\n");
if (inputNum == 0) {
child1name += child1name.length() > 0 ? ", " : "";
- child1name += source.getOptimizerNode().getPactContract().getName();
+ child1name += source.getOptimizerNode().getOperator().getName();
} else if (inputNum == 1) {
child2name += child2name.length() > 0 ? ", " : "";
- child2name = source.getOptimizerNode().getPactContract().getName();
+ child2name = source.getOptimizerNode().getOperator().getName();
}
// output predecessor id
@@ -310,7 +310,7 @@ public class PlanJSONDumpGenerator {
// output shipping strategy and channel type
final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
- ((PactConnection) inConn).getShipStrategy();
+ ((DagConnection) inConn).getShipStrategy();
String shipStrategy = null;
if (shipType != null) {
@@ -588,8 +588,8 @@ public class PlanJSONDumpGenerator {
}
// output the node compiler hints
- if (n.getPactContract().getCompilerHints() != null) {
- CompilerHints hints = n.getPactContract().getCompilerHints();
+ if (n.getOperator().getCompilerHints() != null) {
+ CompilerHints hints = n.getOperator().getCompilerHints();
CompilerHints defaults = new CompilerHints();
String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize());
[44/53] [abbrv] flink git commit: [optimizer] Moved optimizer graph
traversals to dedicated classes to simplify Optimizer class.
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
new file mode 100644
index 0000000..cd8766c
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the various traversals over the program plan and the
+ * optimizer DAG (directed acyclic graph) that are made in the course of
+ * the optimization.
+ *
+ * The traversals are mostly implemented as a {@link org.apache.flink.util.Visitor} that
+ * traversed the program flow.
+ */
+package org.apache.flink.optimizer.traversals;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
index fe33635..1a14be5 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
@@ -25,7 +25,6 @@ import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.dag.OptimizerNode;
import org.apache.flink.optimizer.dag.SingleInputNode;
@@ -37,6 +36,9 @@ import org.apache.flink.optimizer.testfunctions.IdentityFlatMapper;
import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
+import org.apache.flink.optimizer.traversals.BranchesVisitor;
+import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
+import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
import org.junit.Test;
import java.util.Iterator;
@@ -292,8 +294,8 @@ public class PipelineBreakingTest {
}
private static List<DataSinkNode> convertPlan(Plan p) {
- Optimizer.GraphCreatingVisitor dagCreator =
- new Optimizer.GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
+ GraphCreatingVisitor dagCreator =
+ new GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
// create the DAG
p.accept(dagCreator);
@@ -312,8 +314,8 @@ public class PipelineBreakingTest {
rootNode = new SinkJoiner(rootNode, iter.next());
}
}
- rootNode.accept(new Optimizer.IdAndEstimatesVisitor(null));
- rootNode.accept(new Optimizer.BranchesVisitor());
+ rootNode.accept(new IdAndEstimatesVisitor(null));
+ rootNode.accept(new BranchesVisitor());
return sinks;
}
[08/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
new file mode 100644
index 0000000..565d992
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.optimizer.costs.DefaultCostEstimator;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.util.OperatingSystem;
+import org.apache.flink.util.Visitor;
+import org.junit.Before;
+
+/**
+ * Base class for Optimizer tests. Offers utility methods to trigger optimization
+ * of a program and to fetch the nodes in an optimizer plan that correspond
+ * the the node in the program plan.
+ */
+public abstract class CompilerTestBase implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ protected static final String IN_FILE = OperatingSystem.isWindows() ? "file:/c:/" : "file:///dev/random";
+
+ protected static final String OUT_FILE = OperatingSystem.isWindows() ? "file:/c:/" : "file:///dev/null";
+
+ protected static final int DEFAULT_PARALLELISM = 8;
+
+ private static final String CACHE_KEY = "cachekey";
+
+ // ------------------------------------------------------------------------
+
+ protected transient DataStatistics dataStats;
+
+ protected transient Optimizer withStatsCompiler;
+
+ protected transient Optimizer noStatsCompiler;
+
+ private transient int statCounter;
+
+ // ------------------------------------------------------------------------
+
+ @Before
+ public void setup() {
+ this.dataStats = new DataStatistics();
+ this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator());
+ this.withStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator());
+ this.noStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
+ }
+
+ // ------------------------------------------------------------------------
+
+ public OptimizedPlan compileWithStats(Plan p) {
+ return this.withStatsCompiler.compile(p);
+ }
+
+ public OptimizedPlan compileNoStats(Plan p) {
+ return this.noStatsCompiler.compile(p);
+ }
+
+ public void setSourceStatistics(GenericDataSourceBase<?, ?> source, long size, float recordWidth) {
+ setSourceStatistics(source, new FileBaseStatistics(Long.MAX_VALUE, size, recordWidth));
+ }
+
+ public void setSourceStatistics(GenericDataSourceBase<?, ?> source, FileBaseStatistics stats) {
+ final String key = CACHE_KEY + this.statCounter++;
+ this.dataStats.cacheBaseStatistics(stats, key);
+ source.setStatisticsKey(key);
+ }
+
+ // ------------------------------------------------------------------------
+
+ public static OptimizerPlanNodeResolver getOptimizerPlanNodeResolver(OptimizedPlan plan) {
+ return new OptimizerPlanNodeResolver(plan);
+ }
+
+ public static final class OptimizerPlanNodeResolver {
+
+ private final Map<String, ArrayList<PlanNode>> map;
+
+ OptimizerPlanNodeResolver(OptimizedPlan p) {
+ HashMap<String, ArrayList<PlanNode>> map = new HashMap<String, ArrayList<PlanNode>>();
+
+ for (PlanNode n : p.getAllNodes()) {
+ Operator<?> c = n.getOriginalOptimizerNode().getOperator();
+ String name = c.getName();
+
+ ArrayList<PlanNode> list = map.get(name);
+ if (list == null) {
+ list = new ArrayList<PlanNode>(2);
+ map.put(name, list);
+ }
+
+ // check whether this node is a child of a node with the same contract (aka combiner)
+ boolean shouldAdd = true;
+ for (Iterator<PlanNode> iter = list.iterator(); iter.hasNext();) {
+ PlanNode in = iter.next();
+ if (in.getOriginalOptimizerNode().getOperator() == c) {
+ // is this the child or is our node the child
+ if (in instanceof SingleInputPlanNode && n instanceof SingleInputPlanNode) {
+ SingleInputPlanNode thisNode = (SingleInputPlanNode) n;
+ SingleInputPlanNode otherNode = (SingleInputPlanNode) in;
+
+ if (thisNode.getPredecessor() == otherNode) {
+ // other node is child, remove it
+ iter.remove();
+ } else if (otherNode.getPredecessor() == thisNode) {
+ shouldAdd = false;
+ }
+ } else {
+ throw new RuntimeException("Unrecodnized case in test.");
+ }
+ }
+ }
+
+ if (shouldAdd) {
+ list.add(n);
+ }
+ }
+
+ this.map = map;
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public <T extends PlanNode> T getNode(String name) {
+ List<PlanNode> nodes = this.map.get(name);
+ if (nodes == null || nodes.isEmpty()) {
+ throw new RuntimeException("No node found with the given name.");
+ } else if (nodes.size() != 1) {
+ throw new RuntimeException("Multiple nodes found with the given name.");
+ } else {
+ return (T) nodes.get(0);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
+ List<PlanNode> nodes = this.map.get(name);
+ if (nodes == null || nodes.isEmpty()) {
+ throw new RuntimeException("No node found with the given name and stub class.");
+ } else {
+ PlanNode found = null;
+ for (PlanNode node : nodes) {
+ if (node.getClass() == stubClass) {
+ if (found == null) {
+ found = node;
+ } else {
+ throw new RuntimeException("Multiple nodes found with the given name and stub class.");
+ }
+ }
+ }
+ if (found == null) {
+ throw new RuntimeException("No node found with the given name and stub class.");
+ } else {
+ return (T) found;
+ }
+ }
+ }
+
+ public List<PlanNode> getNodes(String name) {
+ List<PlanNode> nodes = this.map.get(name);
+ if (nodes == null || nodes.isEmpty()) {
+ throw new RuntimeException("No node found with the given name.");
+ } else {
+ return new ArrayList<PlanNode>(nodes);
+ }
+ }
+ }
+
+ /**
+ * Collects all DataSources of a plan to add statistics
+ *
+ */
+ public static class SourceCollectorVisitor implements Visitor<Operator<?>> {
+
+ protected final List<GenericDataSourceBase<?, ?>> sources = new ArrayList<GenericDataSourceBase<?, ?>>(4);
+
+ @Override
+ public boolean preVisit(Operator<?> visitable) {
+
+ if(visitable instanceof GenericDataSourceBase) {
+ sources.add((GenericDataSourceBase<?, ?>) visitable);
+ }
+ else if(visitable instanceof BulkIterationBase) {
+ ((BulkIterationBase<?>) visitable).getNextPartialSolution().accept(this);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void postVisit(Operator<?> visitable) {}
+
+ public List<GenericDataSourceBase<?, ?>> getSources() {
+ return this.sources;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
new file mode 100644
index 0000000..b17e777
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.optimizer;
+
+import org.junit.Assert;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.JoinOperator;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyMatchStub;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityMap;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.util.Visitor;
+import org.junit.Test;
+
+/**
+ * Tests in this class:
+ * <ul>
+ * <li>Tests that check the correct handling of the properties and strategies in the case where the degree of
+ * parallelism between tasks is increased or decreased.
+ * </ul>
+ */
+@SuppressWarnings({"serial", "deprecation"})
+public class DOPChangeTest extends CompilerTestBase {
+
+ /**
+ * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
+ *
+ * Increases DOP between 1st reduce and 2nd map, so the hash partitioning from 1st reduce is not reusable.
+ * Expected to re-establish partitioning between reduce and map, via hash, because random is a full network
+ * transit as well.
+ */
+ @Test
+ public void checkPropertyHandlingWithIncreasingGlobalParallelism1() {
+ final int degOfPar = DEFAULT_PARALLELISM;
+
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+ source.setDegreeOfParallelism(degOfPar);
+
+ MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
+ map1.setDegreeOfParallelism(degOfPar);
+ map1.setInput(source);
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
+ reduce1.setDegreeOfParallelism(degOfPar);
+ reduce1.setInput(map1);
+
+ MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
+ map2.setDegreeOfParallelism(degOfPar * 2);
+ map2.setInput(reduce1);
+
+ ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
+ reduce2.setDegreeOfParallelism(degOfPar * 2);
+ reduce2.setInput(map2);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setDegreeOfParallelism(degOfPar * 2);
+ sink.setInput(reduce2);
+
+ Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
+ // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
+ // mapper respectively reducer
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
+ SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
+
+ ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
+ ShipStrategyType redIn = red2Node.getInput().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, mapIn);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, redIn);
+ }
+
+ /**
+ * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
+ *
+ * Increases DOP between 2nd map and 2nd reduce, so the hash partitioning from 1st reduce is not reusable.
+ * Expected to re-establish partitioning between map and reduce (hash).
+ */
+ @Test
+ public void checkPropertyHandlingWithIncreasingGlobalParallelism2() {
+ final int degOfPar = DEFAULT_PARALLELISM;
+
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+ source.setDegreeOfParallelism(degOfPar);
+
+ MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
+ map1.setDegreeOfParallelism(degOfPar);
+ map1.setInput(source);
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
+ reduce1.setDegreeOfParallelism(degOfPar);
+ reduce1.setInput(map1);
+
+ MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
+ map2.setDegreeOfParallelism(degOfPar);
+ map2.setInput(reduce1);
+
+ ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
+ reduce2.setDegreeOfParallelism(degOfPar * 2);
+ reduce2.setInput(map2);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setDegreeOfParallelism(degOfPar * 2);
+ sink.setInput(reduce2);
+
+ Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
+ // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
+ // mapper respectively reducer
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
+ SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
+
+ ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
+ ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, mapIn);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, reduceIn);
+ }
+
+ /**
+ * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
+ *
+ * Increases DOP between 1st reduce and 2nd map, such that more tasks are on one instance.
+ * Expected to re-establish partitioning between map and reduce via a local hash.
+ */
+ @Test
+ public void checkPropertyHandlingWithIncreasingLocalParallelism() {
+ final int degOfPar = 2 * DEFAULT_PARALLELISM;
+
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+ source.setDegreeOfParallelism(degOfPar);
+
+ MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
+ map1.setDegreeOfParallelism(degOfPar);
+ map1.setInput(source);
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
+ reduce1.setDegreeOfParallelism(degOfPar);
+ reduce1.setInput(map1);
+
+ MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
+ map2.setDegreeOfParallelism(degOfPar * 2);
+ map2.setInput(reduce1);
+
+ ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
+ reduce2.setDegreeOfParallelism(degOfPar * 2);
+ reduce2.setInput(map2);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setDegreeOfParallelism(degOfPar * 2);
+ sink.setInput(reduce2);
+
+ Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
+ // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
+ // mapper respectively reducer
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
+ SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
+
+ ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
+ ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
+
+ Assert.assertTrue("Invalid ship strategy for an operator.",
+ (ShipStrategyType.PARTITION_RANDOM == mapIn && ShipStrategyType.PARTITION_HASH == reduceIn) ||
+ (ShipStrategyType.PARTITION_HASH == mapIn && ShipStrategyType.FORWARD == reduceIn));
+ }
+
+
+
+ @Test
+ public void checkPropertyHandlingWithDecreasingDegreeOfParallelism() {
+ final int degOfPar = DEFAULT_PARALLELISM;
+
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+ source.setDegreeOfParallelism(degOfPar * 2);
+
+ MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
+ map1.setDegreeOfParallelism(degOfPar * 2);
+ map1.setInput(source);
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
+ reduce1.setDegreeOfParallelism(degOfPar * 2);
+ reduce1.setInput(map1);
+
+ MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
+ map2.setDegreeOfParallelism(degOfPar);
+ map2.setInput(reduce1);
+
+ ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
+ reduce2.setDegreeOfParallelism(degOfPar);
+ reduce2.setInput(map2);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setDegreeOfParallelism(degOfPar);
+ sink.setInput(reduce2);
+
+ Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
+ // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
+ // mapper respectively reducer
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
+ SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
+
+ Assert.assertTrue("The no sorting local strategy.",
+ LocalStrategy.SORT == red2Node.getInput().getLocalStrategy() ||
+ LocalStrategy.SORT == map2Node.getInput().getLocalStrategy());
+
+ Assert.assertTrue("The no partitioning ship strategy.",
+ ShipStrategyType.PARTITION_HASH == red2Node.getInput().getShipStrategy() ||
+ ShipStrategyType.PARTITION_HASH == map2Node.getInput().getShipStrategy());
+ }
+
+ /**
+ * Checks that re-partitioning happens when the inputs of a two-input contract have different DOPs.
+ *
+ * Test Plan:
+ * <pre>
+ *
+ * (source) -> reduce -\
+ * Match -> (sink)
+ * (source) -> reduce -/
+ *
+ * </pre>
+ *
+ */
+ @Test
+ public void checkPropertyHandlingWithTwoInputs() {
+ // construct the plan
+
+ FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
+ FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
+
+ ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(sourceA)
+ .build();
+ ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(sourceB)
+ .build();
+
+ JoinOperator mat = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
+ .input1(redA)
+ .input2(redB)
+ .build();
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat);
+
+ sourceA.setDegreeOfParallelism(5);
+ sourceB.setDegreeOfParallelism(7);
+ redA.setDegreeOfParallelism(5);
+ redB.setDegreeOfParallelism(7);
+
+ mat.setDegreeOfParallelism(5);
+
+ sink.setDegreeOfParallelism(5);
+
+
+ // return the PACT plan
+ Plan plan = new Plan(sink, "Partition on DoP Change");
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+
+ //Compile plan to verify that no error is thrown
+ jobGen.compileJobGraph(oPlan);
+
+ oPlan.accept(new Visitor<PlanNode>() {
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (visitable instanceof DualInputPlanNode) {
+ DualInputPlanNode node = (DualInputPlanNode) visitable;
+ Channel c1 = node.getInput1();
+ Channel c2 = node.getInput2();
+
+ Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy());
+ Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy());
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+ // DO NOTHING
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
new file mode 100644
index 0000000..aaee975
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class DisjointDataFlowsTest extends CompilerTestBase {
+
+ @Test
+ public void testDisjointFlows() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // generate two different flows
+ env.generateSequence(1, 10).print();
+ env.generateSequence(1, 10).print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
new file mode 100644
index 0000000..34aa9f8
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.DistinctOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+@SuppressWarnings("serial")
+public class DistinctCompilationTest extends CompilerTestBase implements java.io.Serializable {
+
+ @Test
+ public void testDistinctPlain() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .distinct().name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(0, 1), reduceNode.getKeys(0));
+ assertEquals(new FieldList(0, 1), combineNode.getKeys(0));
+ assertEquals(new FieldList(0, 1), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDistinctWithSelectorFunctionKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .distinct(new KeySelector<Tuple2<String,Double>, String>() {
+ public String getKey(Tuple2<String, Double> value) { return value.f0; }
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // get the key extractors and projectors
+ SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
+ SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, keyExtractor.getInput().getSource());
+ assertEquals(keyProjector, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(0), reduceNode.getKeys(0));
+ assertEquals(new FieldList(0), combineNode.getKeys(0));
+ assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDistinctWithFieldPositionKeyCombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ DistinctOperator<Tuple2<String, Double>> reduced = data
+ .distinct(1).name("reducer");
+
+ reduced.print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(1), reduceNode.getKeys(0));
+ assertEquals(new FieldList(1), combineNode.getKeys(0));
+ assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+}
\ No newline at end of file
[04/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
new file mode 100644
index 0000000..0273659
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial","unchecked"})
+public class BinaryCustomPartitioningCompatibilityTest extends CompilerTestBase {
+
+ @Test
+ public void testCompatiblePartitioningJoin() {
+ try {
+ final Partitioner<Long> partitioner = new Partitioner<Long>() {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input1.partitionCustom(partitioner, 1)
+ .join(input2.partitionCustom(partitioner, 0))
+ .where(1).equalTo(0)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode partitioner1 = (SingleInputPlanNode) join.getInput1().getSource();
+ SingleInputPlanNode partitioner2 = (SingleInputPlanNode) join.getInput2().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, join.getInput2().getShipStrategy());
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner1.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner2.getInput().getShipStrategy());
+ assertEquals(partitioner, partitioner1.getInput().getPartitioner());
+ assertEquals(partitioner, partitioner2.getInput().getPartitioner());
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCompatiblePartitioningCoGroup() {
+ try {
+ final Partitioner<Long> partitioner = new Partitioner<Long>() {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ };
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input1.partitionCustom(partitioner, 1)
+ .coGroup(input2.partitionCustom(partitioner, 0))
+ .where(1).equalTo(0)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode partitioner1 = (SingleInputPlanNode) coGroup.getInput1().getSource();
+ SingleInputPlanNode partitioner2 = (SingleInputPlanNode) coGroup.getInput2().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, coGroup.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, coGroup.getInput2().getShipStrategy());
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner1.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner2.getInput().getShipStrategy());
+ assertEquals(partitioner, partitioner1.getInput().getPartitioner());
+ assertEquals(partitioner, partitioner2.getInput().getPartitioner());
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
new file mode 100644
index 0000000..08f7388
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class CoGroupCustomPartitioningTest extends CompilerTestBase {
+
+ @Test
+ public void testCoGroupWithTuples() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input1
+ .coGroup(input2)
+ .where(1).equalTo(0)
+ .withPartitioner(partitioner)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCoGroupWithTuplesWrongType() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ try {
+ input1
+ .coGroup(input2)
+ .where(1).equalTo(0)
+ .withPartitioner(partitioner);
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCoGroupWithPojos() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ input1
+ .coGroup(input2)
+ .where("b").equalTo("a")
+ .withPartitioner(partitioner)
+ .with(new DummyCoGroupFunction<Pojo2, Pojo3>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCoGroupWithPojosWrongType() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ try {
+ input1
+ .coGroup(input2)
+ .where("a").equalTo("b")
+ .withPartitioner(partitioner);
+
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCoGroupWithKeySelectors() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ input1
+ .coGroup(input2)
+ .where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector())
+ .withPartitioner(partitioner)
+ .with(new DummyCoGroupFunction<Pojo2, Pojo3>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCoGroupWithKeySelectorsWrongType() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ try {
+ input1
+ .coGroup(input2)
+ .where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector())
+ .withPartitioner(partitioner);
+
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIncompatibleHashAndCustomPartitioning() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ DataSet<Tuple3<Long, Long, Long>> partitioned = input
+ .partitionCustom(new Partitioner<Long>() {
+ @Override
+ public int partition(Long key, int numPartitions) { return 0; }
+ }, 0)
+ .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2");
+
+
+ DataSet<Tuple3<Long, Long, Long>> grouped = partitioned
+ .distinct(0, 1)
+ .groupBy(1)
+ .sortGroup(0, Order.ASCENDING)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1");
+
+ grouped
+ .coGroup(partitioned).where(0).equalTo(0)
+ .with(new DummyCoGroupFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, coGroup.getInput1().getShipStrategy());
+ assertTrue(coGroup.getInput2().getShipStrategy() == ShipStrategyType.PARTITION_HASH ||
+ coGroup.getInput2().getShipStrategy() == ShipStrategyType.FORWARD);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ public static class Pojo2 {
+ public int a;
+ public int b;
+ }
+
+ public static class Pojo3 {
+ public int a;
+ public int b;
+ public int c;
+ }
+
+ private static class Pojo2KeySelector implements KeySelector<Pojo2, Integer> {
+ @Override
+ public Integer getKey(Pojo2 value) {
+ return value.a;
+ }
+ }
+
+ private static class Pojo3KeySelector implements KeySelector<Pojo3, Integer> {
+ @Override
+ public Integer getKey(Pojo3 value) {
+ return value.b;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
new file mode 100644
index 0000000..9fd676f
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class CustomPartitioningGlobalOptimizationTest extends CompilerTestBase {
+
+ @Test
+ public void testJoinReduceCombination() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ DataSet<Tuple3<Long, Long, Long>> joined = input1.join(input2)
+ .where(1).equalTo(0)
+ .projectFirst(0, 1)
+ .<Tuple3<Long, Long, Long>>projectSecond(2)
+ .withPartitioner(partitioner);
+
+ joined.groupBy(1).withPartitioner(partitioner)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+
+ assertTrue("Reduce is not chained, property reuse does not happen",
+ reducer.getInput().getSource() instanceof DualInputPlanNode);
+
+ DualInputPlanNode join = (DualInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+
+ assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
new file mode 100644
index 0000000..d397ea2
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityPartitionerMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class CustomPartitioningTest extends CompilerTestBase {
+
+ @Test
+ public void testPartitionTuples() {
+ try {
+ final Partitioner<Integer> part = new TestPartitionerInt();
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer,Integer>(0, 0))
+ .rebalance();
+
+ data
+ .partitionCustom(part, 0)
+ .mapPartition(new IdentityPartitionerMapper<Tuple2<Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode partitioner = (SingleInputPlanNode) mapper.getInput().getSource();
+ SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(parallelism, sink.getParallelism());
+
+ assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
+ assertEquals(parallelism, mapper.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
+ assertEquals(part, partitioner.getInput().getPartitioner());
+ assertEquals(parallelism, partitioner.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
+ assertEquals(parallelism, balancer.getParallelism());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionTuplesInvalidType() {
+ try {
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer,Integer>(0, 0))
+ .rebalance();
+
+ try {
+ data
+ .partitionCustom(new TestPartitionerLong(), 0);
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionPojo() {
+ try {
+ final Partitioner<Integer> part = new TestPartitionerInt();
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Pojo> data = env.fromElements(new Pojo())
+ .rebalance();
+
+ data
+ .partitionCustom(part, "a")
+ .mapPartition(new IdentityPartitionerMapper<Pojo>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode partitioner = (SingleInputPlanNode) mapper.getInput().getSource();
+ SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(parallelism, sink.getParallelism());
+
+ assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
+ assertEquals(parallelism, mapper.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
+ assertEquals(part, partitioner.getInput().getPartitioner());
+ assertEquals(parallelism, partitioner.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
+ assertEquals(parallelism, balancer.getParallelism());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionPojoInvalidType() {
+ try {
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Pojo> data = env.fromElements(new Pojo())
+ .rebalance();
+
+ try {
+ data
+ .partitionCustom(new TestPartitionerLong(), "a");
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionKeySelector() {
+ try {
+ final Partitioner<Integer> part = new TestPartitionerInt();
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Pojo> data = env.fromElements(new Pojo())
+ .rebalance();
+
+ data
+ .partitionCustom(part, new TestKeySelectorInt<Pojo>())
+ .mapPartition(new IdentityPartitionerMapper<Pojo>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode keyRemover = (SingleInputPlanNode) mapper.getInput().getSource();
+ SingleInputPlanNode partitioner = (SingleInputPlanNode) keyRemover.getInput().getSource();
+ SingleInputPlanNode keyExtractor = (SingleInputPlanNode) partitioner.getInput().getSource();
+ SingleInputPlanNode balancer = (SingleInputPlanNode) keyExtractor.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(parallelism, sink.getParallelism());
+
+ assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
+ assertEquals(parallelism, mapper.getParallelism());
+
+ assertEquals(ShipStrategyType.FORWARD, keyRemover.getInput().getShipStrategy());
+ assertEquals(parallelism, keyRemover.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
+ assertEquals(part, partitioner.getInput().getPartitioner());
+ assertEquals(parallelism, partitioner.getParallelism());
+
+ assertEquals(ShipStrategyType.FORWARD, keyExtractor.getInput().getShipStrategy());
+ assertEquals(parallelism, keyExtractor.getParallelism());
+
+ assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
+ assertEquals(parallelism, balancer.getParallelism());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionKeySelectorInvalidType() {
+ try {
+ final Partitioner<Integer> part = (Partitioner<Integer>) (Partitioner<?>) new TestPartitionerLong();
+ final int parallelism = 4;
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(parallelism);
+
+ DataSet<Pojo> data = env.fromElements(new Pojo())
+ .rebalance();
+
+ try {
+ data
+ .partitionCustom(part, new TestKeySelectorInt<Pojo>());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class Pojo {
+ public int a;
+ public int b;
+ }
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestKeySelectorInt<T> implements KeySelector<T, Integer> {
+ @Override
+ public Integer getKey(T value) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
new file mode 100644
index 0000000..360487b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class GroupingKeySelectorTranslationTest extends CompilerTestBase {
+
+ @Test
+ public void testCustomPartitioningKeySelectorReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
+ .withPartitioner(new TestPartitionerInt())
+ .reduce(new DummyReducer<Tuple2<Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode keyRemovingMapper = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) keyRemovingMapper.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, keyRemovingMapper.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningKeySelectorGroupReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
+ .withPartitioner(new TestPartitionerInt())
+ .reduceGroup(new IdentityGroupReducer<Tuple2<Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningKeySelectorGroupReduceSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(new TestKeySelector<Tuple3<Integer,Integer,Integer>>())
+ .withPartitioner(new TestPartitionerInt())
+ .sortGroup(new TestKeySelector<Tuple3<Integer, Integer, Integer>>(), Order.ASCENDING)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Integer,Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningKeySelectorInvalidType() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data
+ .groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
+ .withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningKeySelectorInvalidTypeSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data
+ .groupBy(new TestKeySelector<Tuple3<Integer,Integer,Integer>>())
+ .sortGroup(1, Order.ASCENDING)
+ .withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleRejectCompositeKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data
+ .groupBy(new TestBinaryKeySelector<Tuple3<Integer,Integer,Integer>>())
+ .withPartitioner(new TestPartitionerInt());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestKeySelector<T extends Tuple> implements KeySelector<T, Integer> {
+ @Override
+ public Integer getKey(T value) {
+ return value.getField(0);
+ }
+ }
+
+ private static class TestBinaryKeySelector<T extends Tuple> implements KeySelector<T, Tuple2<Integer, Integer>> {
+ @Override
+ public Tuple2<Integer, Integer> getKey(T value) {
+ return new Tuple2<Integer, Integer>(value.<Integer>getField(0), value.<Integer>getField(1));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
new file mode 100644
index 0000000..8cd4809
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class GroupingPojoTranslationTest extends CompilerTestBase {
+
+ @Test
+ public void testCustomPartitioningTupleReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> data = env.fromElements(new Pojo2())
+ .rebalance().setParallelism(4);
+
+ data.groupBy("a").withPartitioner(new TestPartitionerInt())
+ .reduce(new DummyReducer<Pojo2>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> data = env.fromElements(new Pojo2())
+ .rebalance().setParallelism(4);
+
+ data.groupBy("a").withPartitioner(new TestPartitionerInt())
+ .reduceGroup(new IdentityGroupReducer<Pojo2>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduceSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo3> data = env.fromElements(new Pojo3())
+ .rebalance().setParallelism(4);
+
+ data.groupBy("a").withPartitioner(new TestPartitionerInt())
+ .sortGroup("b", Order.ASCENDING)
+ .reduceGroup(new IdentityGroupReducer<Pojo3>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduceSorted2() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo4> data = env.fromElements(new Pojo4())
+ .rebalance().setParallelism(4);
+
+ data.groupBy("a").withPartitioner(new TestPartitionerInt())
+ .sortGroup("b", Order.ASCENDING)
+ .sortGroup("c", Order.DESCENDING)
+ .reduceGroup(new IdentityGroupReducer<Pojo4>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleInvalidType() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> data = env.fromElements(new Pojo2())
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy("a").withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleInvalidTypeSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo3> data = env.fromElements(new Pojo3())
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy("a")
+ .sortGroup("b", Order.ASCENDING)
+ .withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleRejectCompositeKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> data = env.fromElements(new Pojo2())
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy("a", "b")
+ .withPartitioner(new TestPartitionerInt());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class Pojo2 {
+ public int a;
+ public int b;
+
+ }
+
+ public static class Pojo3 {
+ public int a;
+ public int b;
+ public int c;
+ }
+
+ public static class Pojo4 {
+ public int a;
+ public int b;
+ public int c;
+ public int d;
+ }
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
new file mode 100644
index 0000000..779b8e5
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class GroupingTupleTranslationTest extends CompilerTestBase {
+
+ @Test
+ public void testCustomPartitioningTupleAgg() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(0).withPartitioner(new TestPartitionerInt())
+ .sum(1)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(0).withPartitioner(new TestPartitionerInt())
+ .reduce(new DummyReducer<Tuple2<Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(0).withPartitioner(new TestPartitionerInt())
+ .reduceGroup(new IdentityGroupReducer<Tuple2<Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduceSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(0).withPartitioner(new TestPartitionerInt())
+ .sortGroup(1, Order.ASCENDING)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Integer,Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleGroupReduceSorted2() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple4<Integer,Integer,Integer, Integer>> data = env.fromElements(new Tuple4<Integer,Integer,Integer,Integer>(0, 0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ data.groupBy(0).withPartitioner(new TestPartitionerInt())
+ .sortGroup(1, Order.ASCENDING)
+ .sortGroup(2, Order.DESCENDING)
+ .reduceGroup(new IdentityGroupReducer<Tuple4<Integer,Integer,Integer,Integer>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleInvalidType() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy(0).withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleInvalidTypeSorted() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy(0)
+ .sortGroup(1, Order.ASCENDING)
+ .withPartitioner(new TestPartitionerLong());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningTupleRejectCompositeKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
+ .rebalance().setParallelism(4);
+
+ try {
+ data.groupBy(0, 1)
+ .withPartitioner(new TestPartitionerInt());
+ fail("Should throw an exception");
+ }
+ catch (InvalidProgramException e) {}
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
new file mode 100644
index 0000000..eae40cf
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.custompartition;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class JoinCustomPartitioningTest extends CompilerTestBase {
+
+ @Test
+ public void testJoinWithTuples() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST).where(1).equalTo(0).withPartitioner(partitioner)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJoinWithTuplesWrongType() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ try {
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST).where(1).equalTo(0)
+ .withPartitioner(partitioner);
+
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJoinWithPojos() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST)
+ .where("b").equalTo("a").withPartitioner(partitioner)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJoinWithPojosWrongType() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ try {
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST)
+ .where("a").equalTo("b")
+ .withPartitioner(partitioner);
+
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJoinWithKeySelectors() {
+ try {
+ final Partitioner<Integer> partitioner = new TestPartitionerInt();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST)
+ .where(new Pojo2KeySelector())
+ .equalTo(new Pojo3KeySelector())
+ .withPartitioner(partitioner)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
+ assertEquals(partitioner, join.getInput1().getPartitioner());
+ assertEquals(partitioner, join.getInput2().getPartitioner());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testJoinWithKeySelectorsWrongType() {
+ try {
+ final Partitioner<Long> partitioner = new TestPartitionerLong();
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
+ DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
+
+ try {
+ input1
+ .join(input2, JoinHint.REPARTITION_HASH_FIRST)
+ .where(new Pojo2KeySelector())
+ .equalTo(new Pojo3KeySelector())
+ .withPartitioner(partitioner);
+
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIncompatibleHashAndCustomPartitioning() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ DataSet<Tuple3<Long, Long, Long>> partitioned = input
+ .partitionCustom(new Partitioner<Long>() {
+ @Override
+ public int partition(Long key, int numPartitions) { return 0; }
+ }, 0)
+ .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2");
+
+
+ DataSet<Tuple3<Long, Long, Long>> grouped = partitioned
+ .distinct(0, 1)
+ .groupBy(1)
+ .sortGroup(0, Order.ASCENDING)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1");
+
+ grouped
+ .join(partitioned, JoinHint.REPARTITION_HASH_FIRST).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple3<Long,Long,Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, coGroup.getInput1().getShipStrategy());
+ assertTrue(coGroup.getInput2().getShipStrategy() == ShipStrategyType.PARTITION_HASH ||
+ coGroup.getInput2().getShipStrategy() == ShipStrategyType.FORWARD);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static class TestPartitionerInt implements Partitioner<Integer> {
+ @Override
+ public int partition(Integer key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ private static class TestPartitionerLong implements Partitioner<Long> {
+ @Override
+ public int partition(Long key, int numPartitions) {
+ return 0;
+ }
+ }
+
+ public static class Pojo2 {
+ public int a;
+ public int b;
+ }
+
+ public static class Pojo3 {
+ public int a;
+ public int b;
+ public int c;
+ }
+
+ private static class Pojo2KeySelector implements KeySelector<Pojo2, Integer> {
+ @Override
+ public Integer getKey(Pojo2 value) {
+ return value.a;
+ }
+ }
+
+ private static class Pojo3KeySelector implements KeySelector<Pojo3, Integer> {
+ @Override
+ public Integer getKey(Pojo3 value) {
+ return value.b;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
new file mode 100644
index 0000000..cb4bd78
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataexchange;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityFlatMapper;
+import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
+import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.junit.Test;
+
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * This test checks the correct assignment of the DataExchangeMode to
+ * connections for programs that branch, and re-join those branches.
+ *
+ * <pre>
+ * /-> (sink)
+ * /
+ * /-> (reduce) -+ /-> (flatmap) -> (sink)
+ * / \ /
+ * (source) -> (map) - (join) -+-----\
+ * \ / \
+ * \-> (filter) -+ \
+ * \ (co group) -> (sink)
+ * \ /
+ * \-> (reduce) - /
+ * </pre>
+ */
+@SuppressWarnings("serial")
+public class DataExchangeModeClosedBranchingTest extends CompilerTestBase {
+
+ @Test
+ public void testPipelinedForced() {
+ // PIPELINED_FORCED should result in pipelining all the way
+ verifyBranchingJoiningPlan(ExecutionMode.PIPELINED_FORCED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
+ DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
+ }
+
+ @Test
+ public void testPipelined() {
+ // PIPELINED should result in pipelining all the way
+ verifyBranchingJoiningPlan(ExecutionMode.PIPELINED,
+ DataExchangeMode.PIPELINED, // to map
+ DataExchangeMode.PIPELINED, // to combiner connections are pipelined
+ DataExchangeMode.BATCH, // to reduce
+ DataExchangeMode.BATCH, // to filter
+ DataExchangeMode.PIPELINED, // to sink after reduce
+ DataExchangeMode.PIPELINED, // to join (first input)
+ DataExchangeMode.BATCH, // to join (second input)
+ DataExchangeMode.PIPELINED, // combiner connections are pipelined
+ DataExchangeMode.BATCH, // to other reducer
+ DataExchangeMode.PIPELINED, // to flatMap
+ DataExchangeMode.PIPELINED, // to sink after flatMap
+ DataExchangeMode.PIPELINED, // to coGroup (first input)
+ DataExchangeMode.PIPELINED, // to coGroup (second input)
+ DataExchangeMode.PIPELINED // to sink after coGroup
+ );
+ }
+
+ @Test
+ public void testBatch() {
+ // BATCH should result in batching the shuffle all the way
+ verifyBranchingJoiningPlan(ExecutionMode.BATCH,
+ DataExchangeMode.PIPELINED, // to map
+ DataExchangeMode.PIPELINED, // to combiner connections are pipelined
+ DataExchangeMode.BATCH, // to reduce
+ DataExchangeMode.BATCH, // to filter
+ DataExchangeMode.PIPELINED, // to sink after reduce
+ DataExchangeMode.BATCH, // to join (first input)
+ DataExchangeMode.BATCH, // to join (second input)
+ DataExchangeMode.PIPELINED, // combiner connections are pipelined
+ DataExchangeMode.BATCH, // to other reducer
+ DataExchangeMode.PIPELINED, // to flatMap
+ DataExchangeMode.PIPELINED, // to sink after flatMap
+ DataExchangeMode.BATCH, // to coGroup (first input)
+ DataExchangeMode.BATCH, // to coGroup (second input)
+ DataExchangeMode.PIPELINED // to sink after coGroup
+ );
+ }
+
+ @Test
+ public void testBatchForced() {
+ // BATCH_FORCED should result in batching all the way
+ verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
+ DataExchangeMode.BATCH, // to map
+ DataExchangeMode.PIPELINED, // to combiner connections are pipelined
+ DataExchangeMode.BATCH, // to reduce
+ DataExchangeMode.BATCH, // to filter
+ DataExchangeMode.BATCH, // to sink after reduce
+ DataExchangeMode.BATCH, // to join (first input)
+ DataExchangeMode.BATCH, // to join (second input)
+ DataExchangeMode.PIPELINED, // combiner connections are pipelined
+ DataExchangeMode.BATCH, // to other reducer
+ DataExchangeMode.BATCH, // to flatMap
+ DataExchangeMode.BATCH, // to sink after flatMap
+ DataExchangeMode.BATCH, // to coGroup (first input)
+ DataExchangeMode.BATCH, // to coGroup (second input)
+ DataExchangeMode.BATCH // to sink after coGroup
+ );
+ }
+
+ private void verifyBranchingJoiningPlan(ExecutionMode execMode,
+ DataExchangeMode toMap,
+ DataExchangeMode toReduceCombiner,
+ DataExchangeMode toReduce,
+ DataExchangeMode toFilter,
+ DataExchangeMode toReduceSink,
+ DataExchangeMode toJoin1,
+ DataExchangeMode toJoin2,
+ DataExchangeMode toOtherReduceCombiner,
+ DataExchangeMode toOtherReduce,
+ DataExchangeMode toFlatMap,
+ DataExchangeMode toFlatMapSink,
+ DataExchangeMode toCoGroup1,
+ DataExchangeMode toCoGroup2,
+ DataExchangeMode toCoGroupSink)
+ {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.getConfig().setExecutionMode(execMode);
+
+ DataSet<Tuple2<Long, Long>> data = env.fromElements(33L, 44L)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value) {
+ return new Tuple2<Long, Long>(value, value);
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> reduced = data.groupBy(0).reduce(new SelectOneReducer<Tuple2<Long, Long>>());
+ reduced.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("reduceSink");
+
+ DataSet<Tuple2<Long, Long>> filtered = data.filter(new FilterFunction<Tuple2<Long, Long>>() {
+ @Override
+ public boolean filter(Tuple2<Long, Long> value) throws Exception {
+ return false;
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> joined = reduced.join(filtered)
+ .where(1).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ joined.flatMap(new IdentityFlatMapper<Tuple2<Long, Long>>())
+ .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("flatMapSink");
+
+ joined.coGroup(filtered.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()))
+ .where(0).equalTo(0)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()).name("cgSink");
+
+
+ OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
+
+ SinkPlanNode reduceSink = findSink(optPlan.getDataSinks(), "reduceSink");
+ SinkPlanNode flatMapSink = findSink(optPlan.getDataSinks(), "flatMapSink");
+ SinkPlanNode cgSink = findSink(optPlan.getDataSinks(), "cgSink");
+
+ DualInputPlanNode coGroupNode = (DualInputPlanNode) cgSink.getPredecessor();
+
+ DualInputPlanNode joinNode = (DualInputPlanNode) coGroupNode.getInput1().getSource();
+ SingleInputPlanNode otherReduceNode = (SingleInputPlanNode) coGroupNode.getInput2().getSource();
+ SingleInputPlanNode otherReduceCombinerNode = (SingleInputPlanNode) otherReduceNode.getPredecessor();
+
+ SingleInputPlanNode reduceNode = (SingleInputPlanNode) joinNode.getInput1().getSource();
+ SingleInputPlanNode reduceCombinerNode = (SingleInputPlanNode) reduceNode.getPredecessor();
+ assertEquals(reduceNode, reduceSink.getPredecessor());
+
+ SingleInputPlanNode filterNode = (SingleInputPlanNode) joinNode.getInput2().getSource();
+ assertEquals(filterNode, otherReduceCombinerNode.getPredecessor());
+
+ SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
+ assertEquals(mapNode, reduceCombinerNode.getPredecessor());
+
+ SingleInputPlanNode flatMapNode = (SingleInputPlanNode) flatMapSink.getPredecessor();
+ assertEquals(joinNode, flatMapNode.getPredecessor());
+
+ // verify the data exchange modes
+
+ assertEquals(toReduceSink, reduceSink.getInput().getDataExchangeMode());
+ assertEquals(toFlatMapSink, flatMapSink.getInput().getDataExchangeMode());
+ assertEquals(toCoGroupSink, cgSink.getInput().getDataExchangeMode());
+
+ assertEquals(toCoGroup1, coGroupNode.getInput1().getDataExchangeMode());
+ assertEquals(toCoGroup2, coGroupNode.getInput2().getDataExchangeMode());
+
+ assertEquals(toJoin1, joinNode.getInput1().getDataExchangeMode());
+ assertEquals(toJoin2, joinNode.getInput2().getDataExchangeMode());
+
+ assertEquals(toOtherReduce, otherReduceNode.getInput().getDataExchangeMode());
+ assertEquals(toOtherReduceCombiner, otherReduceCombinerNode.getInput().getDataExchangeMode());
+
+ assertEquals(toFlatMap, flatMapNode.getInput().getDataExchangeMode());
+
+ assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
+ assertEquals(toReduce, reduceNode.getInput().getDataExchangeMode());
+ assertEquals(toReduceCombiner, reduceCombinerNode.getInput().getDataExchangeMode());
+
+ assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
+ for (SinkPlanNode node : collection) {
+ String nodeName = node.getOptimizerNode().getOperator().getName();
+ if (nodeName != null && nodeName.equals(name)) {
+ return node;
+ }
+ }
+
+ throw new IllegalArgumentException("No node with that name was found.");
+ }
+}
[38/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
deleted file mode 100644
index dbe04f4..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.util.Visitor;
-
-/**
- * The Optimizer representation of a data sink.
- */
-public class DataSinkNode extends OptimizerNode {
-
- protected DagConnection input; // The input edge
-
- /**
- * Creates a new DataSinkNode for the given sink operator.
- *
- * @param sink The data sink contract object.
- */
- public DataSinkNode(GenericDataSinkBase<?> sink) {
- super(sink);
- }
-
- // --------------------------------------------------------------------------------------
-
- /**
- * Gets the input of the sink.
- *
- * @return The input connection.
- */
- public DagConnection getInputConnection() {
- return this.input;
- }
-
- /**
- * Gets the predecessor of this node.
- *
- * @return The predecessor, or null, if no predecessor has been set.
- */
- public OptimizerNode getPredecessorNode() {
- if(this.input != null) {
- return input.getSource();
- } else {
- return null;
- }
- }
-
- /**
- * Gets the operator for which this optimizer sink node was created.
- *
- * @return The node's underlying operator.
- */
- @Override
- public GenericDataSinkBase<?> getOperator() {
- return (GenericDataSinkBase<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Data Sink";
- }
-
- @Override
- public List<DagConnection> getIncomingConnections() {
- return Collections.singletonList(this.input);
- }
-
- /**
- * Gets all outgoing connections, which is an empty set for the data sink.
- *
- * @return An empty list.
- */
- @Override
- public List<DagConnection> getOutgoingConnections() {
- return Collections.emptyList();
- }
-
- @Override
- public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
- Operator<?> children = getOperator().getInput();
-
- final OptimizerNode pred;
- final DagConnection conn;
-
- pred = contractToNode.get(children);
- conn = new DagConnection(pred, this, defaultExchangeMode);
-
- // create the connection and add it
- this.input = conn;
- pred.addOutgoingConnection(conn);
- }
-
- /**
- * Computes the estimated outputs for the data sink. Since the sink does not modify anything, it simply
- * copies the output estimates from its direct predecessor.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- final InterestingProperties iProps = new InterestingProperties();
-
- {
- final Ordering partitioning = getOperator().getPartitionOrdering();
- final DataDistribution dataDist = getOperator().getDataDistribution();
- final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
- if (partitioning != null) {
- if(dataDist != null) {
- partitioningProps.setRangePartitioned(partitioning, dataDist);
- } else {
- partitioningProps.setRangePartitioned(partitioning);
- }
- iProps.addGlobalProperties(partitioningProps);
- }
- iProps.addGlobalProperties(partitioningProps);
- }
-
- {
- final Ordering localOrder = getOperator().getLocalOrder();
- final RequestedLocalProperties orderProps = new RequestedLocalProperties();
- if (localOrder != null) {
- orderProps.setOrdering(localOrder);
- }
- iProps.addLocalProperties(orderProps);
- }
-
- this.input.setInterestingProperties(iProps);
- }
-
- // --------------------------------------------------------------------------------------------
- // Branch Handling
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- // we need to track open branches even in the sinks, because they get "closed" when
- // we build a single "root" for the data flow plan
- addClosedBranches(getPredecessorNode().closedBranchingNodes);
- this.openBranches = getPredecessorNode().getBranchesForParent(this.input);
- }
-
- @Override
- protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection parent) {
- // return our own stack of open branches, because nothing is added
- return this.openBranches;
- }
-
- // --------------------------------------------------------------------------------------------
- // Recursive Optimization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- // check if we have a cached version
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- }
-
- // calculate alternative sub-plans for predecessor
- List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
- List<PlanNode> outputPlans = new ArrayList<PlanNode>();
-
- final int dop = getParallelism();
- final int inDop = getPredecessorNode().getParallelism();
-
- final ExecutionMode executionMode = this.input.getDataExchangeMode();
- final boolean dopChange = dop != inDop;
- final boolean breakPipeline = this.input.isBreakingPipeline();
-
- InterestingProperties ips = this.input.getInterestingProperties();
- for (PlanNode p : subPlans) {
- for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
- for (RequestedLocalProperties lp : ips.getLocalProperties()) {
- Channel c = new Channel(p);
- gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
- lp.parameterizeChannel(c);
- c.setRequiredLocalProps(lp);
- c.setRequiredGlobalProps(gp);
-
- // no need to check whether the created properties meet what we need in case
- // of ordering or global ordering, because the only interesting properties we have
- // are what we require
- outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
- }
- }
- }
-
- // cost and prune the plans
- for (PlanNode node : outputPlans) {
- estimator.costOperator(node);
- }
- prunePlanAlternatives(outputPlans);
-
- this.cachedPlans = outputPlans;
- return outputPlans;
- }
-
- // --------------------------------------------------------------------------------------------
- // Function Annotation Handling
- // --------------------------------------------------------------------------------------------
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new EmptySemanticProperties();
- }
-
- // --------------------------------------------------------------------------------------------
- // Miscellaneous
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void accept(Visitor<OptimizerNode> visitor) {
- if (visitor.preVisit(this)) {
- if (getPredecessorNode() != null) {
- getPredecessorNode().accept(visitor);
- } else {
- throw new CompilerException();
- }
- visitor.postVisit(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
deleted file mode 100644
index e4b35b7..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.io.FileInputFormat;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.io.NonParallelInput;
-import org.apache.flink.api.common.io.ReplicatingInputFormat;
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.GenericDataSourceBase.SplitDataProperties;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.util.Visitor;
-
-/**
- * The optimizer's internal representation of a data source.
- */
-public class DataSourceNode extends OptimizerNode {
-
- private final boolean sequentialInput;
-
- private final boolean replicatedInput;
-
- private GlobalProperties gprops;
-
- private LocalProperties lprops;
-
- /**
- * Creates a new DataSourceNode for the given contract.
- *
- * @param pactContract
- * The data source contract object.
- */
- public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
- super(pactContract);
-
- if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
- throw new IllegalArgumentException("Input format has not been set.");
- }
-
- if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
- setDegreeOfParallelism(1);
- this.sequentialInput = true;
- } else {
- this.sequentialInput = false;
- }
-
- this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
- pactContract.getUserCodeWrapper().getUserCodeClass());
-
- this.gprops = new GlobalProperties();
- this.lprops = new LocalProperties();
-
- SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();
-
- if(replicatedInput) {
- this.gprops.setFullyReplicated();
- this.lprops = new LocalProperties();
- } else if (splitProps != null) {
- // configure data properties of data source using split properties
- setDataPropertiesFromSplitProperties(splitProps);
- }
-
- }
-
- /**
- * Gets the contract object for this data source node.
- *
- * @return The contract.
- */
- @Override
- public GenericDataSourceBase<?, ?> getOperator() {
- return (GenericDataSourceBase<?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Data Source";
- }
-
- @Override
- public void setDegreeOfParallelism(int degreeOfParallelism) {
- // if unsplittable, parallelism remains at 1
- if (!this.sequentialInput) {
- super.setDegreeOfParallelism(degreeOfParallelism);
- }
- }
-
- @Override
- public List<DagConnection> getIncomingConnections() {
- return Collections.<DagConnection>emptyList();
- }
-
- @Override
- public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultDataExchangeMode) {}
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // see, if we have a statistics object that can tell us a bit about the file
- if (statistics != null) {
- // instantiate the input format, as this is needed by the statistics
- InputFormat<?, ?> format;
- String inFormatDescription = "<unknown>";
-
- try {
- format = getOperator().getFormatWrapper().getUserCodeObject();
- Configuration config = getOperator().getParameters();
- format.configure(config);
- }
- catch (Throwable t) {
- if (Optimizer.LOG.isWarnEnabled()) {
- Optimizer.LOG.warn("Could not instantiate InputFormat to obtain statistics."
- + " Limited statistics will be available.", t);
- }
- return;
- }
- try {
- inFormatDescription = format.toString();
- }
- catch (Throwable t) {
- // we can ignore this error, as it only prevents us to use a cosmetic string
- }
-
- // first of all, get the statistics from the cache
- final String statisticsKey = getOperator().getStatisticsKey();
- final BaseStatistics cachedStatistics = statistics.getBaseStatistics(statisticsKey);
-
- BaseStatistics bs = null;
- try {
- bs = format.getStatistics(cachedStatistics);
- }
- catch (Throwable t) {
- if (Optimizer.LOG.isWarnEnabled()) {
- Optimizer.LOG.warn("Error obtaining statistics from input format: " + t.getMessage(), t);
- }
- }
-
- if (bs != null) {
- final long len = bs.getTotalInputSize();
- if (len == BaseStatistics.SIZE_UNKNOWN) {
- if (Optimizer.LOG.isInfoEnabled()) {
- Optimizer.LOG.info("Compiler could not determine the size of input '" + inFormatDescription + "'. Using default estimates.");
- }
- }
- else if (len >= 0) {
- this.estimatedOutputSize = len;
- }
-
- final long card = bs.getNumberOfRecords();
- if (card != BaseStatistics.NUM_RECORDS_UNKNOWN) {
- this.estimatedNumRecords = card;
- }
- }
- }
- }
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- // no children, so nothing to compute
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- // because there are no inputs, there are no unclosed branches.
- this.openBranches = Collections.emptyList();
- }
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- }
-
- SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")",
- this.gprops, this.lprops);
-
- if(!replicatedInput) {
- candidate.updatePropertiesWithUniqueSets(getUniqueFields());
-
- final Costs costs = new Costs();
- if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
- this.estimatedOutputSize >= 0) {
- estimator.addFileInputCost(this.estimatedOutputSize, costs);
- }
- candidate.setCosts(costs);
- } else {
- // replicated input
- final Costs costs = new Costs();
- InputFormat<?,?> inputFormat =
- ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
- if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
- this.estimatedOutputSize >= 0) {
- estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs);
- }
- candidate.setCosts(costs);
- }
-
- // since there is only a single plan for the data-source, return a list with that element only
- List<PlanNode> plans = new ArrayList<PlanNode>(1);
- plans.add(candidate);
-
- this.cachedPlans = plans;
- return plans;
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new EmptySemanticProperties();
- }
-
- @Override
- public void accept(Visitor<OptimizerNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-
- private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {
-
- // set global properties
- int[] partitionKeys = splitProps.getSplitPartitionKeys();
- Partitioner<?> partitioner = splitProps.getSplitPartitioner();
-
- if(partitionKeys != null && partitioner != null) {
- this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
- }
- else if(partitionKeys != null) {
- this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
- }
- // set local properties
- int[] groupingKeys = splitProps.getSplitGroupKeys();
- Ordering ordering = splitProps.getSplitOrder();
-
- // more than one split per source tasks possible.
- // adapt split grouping and sorting
- if(ordering != null) {
-
- // sorting falls back to grouping because a source can read multiple,
- // randomly assigned splits
- groupingKeys = ordering.getFieldPositions();
- }
-
- if(groupingKeys != null && partitionKeys != null) {
- // check if grouping is also valid across splits, i.e., whether grouping keys are
- // valid superset of partition keys
- boolean allFieldsIncluded = true;
- for(int i : partitionKeys) {
- boolean fieldIncluded = false;
- for(int j : groupingKeys) {
- if(i == j) {
- fieldIncluded = true;
- break;
- }
- }
- if(!fieldIncluded) {
- allFieldsIncluded = false;
- break;
- }
- }
- if (allFieldsIncluded) {
- this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
- } else {
- this.lprops = new LocalProperties();
- }
-
- } else {
- this.lprops = new LocalProperties();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
deleted file mode 100644
index 482951b..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-/**
- * Methods for operators / connections that provide estimated about data size and
- * characteristics.
- */
-public interface EstimateProvider {
-
- /**
- * Gets the estimated output size from this node.
- *
- * @return The estimated output size.
- */
- long getEstimatedOutputSize();
-
- /**
- * Gets the estimated number of records in the output of this node.
- *
- * @return The estimated number of records.
- */
- long getEstimatedNumRecords();
-
- /**
- * Gets the estimated number of bytes per record.
- *
- * @return The estimated number of bytes per record.
- */
- float getEstimatedAvgWidthPerOutputRecord();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
deleted file mode 100644
index 118ddc8..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.base.FilterOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.FilterDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-/**
- * The optimizer's internal representation of a <i>FlatMap</i> operator node.
- */
-public class FilterNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- public FilterNode(FilterOperatorBase<?, ?> operator) {
- super(operator);
- this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new FilterDescriptor());
- }
-
- @Override
- public FilterOperatorBase<?, ?> getOperator() {
- return (FilterOperatorBase<?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Filter";
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- /**
- * Computes the estimates for the Filter operator. Since it applies a filter on the data we assume a cardinality
- * decrease. To give the system a hint at data decrease, we use a default magic number to indicate a 0.5 decrease.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = (long) (getPredecessorNode().getEstimatedNumRecords() * 0.5);
- this.estimatedOutputSize = (long) (getPredecessorNode().getEstimatedOutputSize() * 0.5);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
deleted file mode 100644
index f713d56..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.FlatMapDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-/**
- * The optimizer's internal representation of a <i>FlatMap</i> operator node.
- */
-public class FlatMapNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- public FlatMapNode(FlatMapOperatorBase<?, ?, ?> operator) {
- super(operator);
-
- this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new FlatMapDescriptor());
- }
-
- @Override
- public FlatMapOperatorBase<?, ?, ?> getOperator() {
- return (FlatMapOperatorBase<?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "FlatMap";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- /**
- * Computes the estimates for the FlatMap operator. Since it un-nests, we assume a cardinality
- * increase. To give the system a hint at data increase, we take a default magic number of a 5 times increase.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords() * 5;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
deleted file mode 100644
index 564c0d3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.AllGroupCombineProperties;
-import org.apache.flink.optimizer.operators.GroupCombineProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The optimizer representation of a <i>GroupCombineNode</i> operation.
- */
-public class GroupCombineNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- /**
- * Creates a new optimizer node for the given operator.
- *
- * @param operator The reduce operation.
- */
- public GroupCombineNode(GroupCombineOperatorBase<?, ?, ?> operator) {
- super(operator);
-
- if (this.keys == null) {
- // case of a key-less reducer. force a parallelism of 1
- setDegreeOfParallelism(1);
- }
-
- this.possibleProperties = initPossibleProperties();
- }
-
- private List<OperatorDescriptorSingle> initPossibleProperties() {
-
- // check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
- Ordering groupOrder = getOperator().getGroupOrder();
- if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
- groupOrder = null;
- }
-
- OperatorDescriptorSingle props = (this.keys == null ?
- new AllGroupCombineProperties() :
- new GroupCombineProperties(this.keys, groupOrder));
-
- return Collections.singletonList(props);
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Gets the operator represented by this optimizer node.
- *
- * @return The operator represented by this optimizer node.
- */
- @Override
- public GroupCombineOperatorBase<?, ?, ?> getOperator() {
- return (GroupCombineOperatorBase<?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "GroupCombine";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- // --------------------------------------------------------------------------------------------
- // Estimates
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // no real estimates possible for a reducer.
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
deleted file mode 100644
index 77acae5..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.operators.AllGroupReduceProperties;
-import org.apache.flink.optimizer.operators.AllGroupWithPartialPreGroupProperties;
-import org.apache.flink.optimizer.operators.GroupReduceProperties;
-import org.apache.flink.optimizer.operators.GroupReduceWithCombineProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.configuration.Configuration;
-
-/**
- * The optimizer representation of a <i>GroupReduce</i> operation.
- */
-public class GroupReduceNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- private GroupReduceNode combinerUtilityNode;
-
- /**
- * Creates a new optimizer node for the given operator.
- *
- * @param operator The reduce operation.
- */
- public GroupReduceNode(GroupReduceOperatorBase<?, ?, ?> operator) {
- super(operator);
-
- if (this.keys == null) {
- // case of a key-less reducer. force a parallelism of 1
- setDegreeOfParallelism(1);
- }
-
- this.possibleProperties = initPossibleProperties(operator.getCustomPartitioner());
- }
-
- public GroupReduceNode(GroupReduceNode reducerToCopyForCombiner) {
- super(reducerToCopyForCombiner);
-
- this.possibleProperties = Collections.emptyList();
- }
-
- private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
- // see if an internal hint dictates the strategy to use
- final Configuration conf = getOperator().getParameters();
- final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
-
- final boolean useCombiner;
- if (localStrategy != null) {
- if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
- useCombiner = false;
- }
- else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
- if (!isCombineable()) {
- Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
- "' requires combinable reduce, but user function is not marked combinable.");
- }
- useCombiner = true;
- } else {
- throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
- }
- } else {
- useCombiner = isCombineable();
- }
-
- // check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
- Ordering groupOrder = null;
- if (getOperator() instanceof GroupReduceOperatorBase) {
- groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getOperator()).getGroupOrder();
- if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
- groupOrder = null;
- }
- }
-
- OperatorDescriptorSingle props = useCombiner ?
- (this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
- (this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));
-
- return Collections.singletonList(props);
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Gets the operator represented by this optimizer node.
- *
- * @return The operator represented by this optimizer node.
- */
- @Override
- public GroupReduceOperatorBase<?, ?, ?> getOperator() {
- return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
- }
-
- /**
- * Checks, whether a combiner function has been given for the function encapsulated
- * by this reduce contract.
- *
- * @return True, if a combiner has been given, false otherwise.
- */
- public boolean isCombineable() {
- return getOperator().isCombinable();
- }
-
- @Override
- public String getName() {
- return "GroupReduce";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- // --------------------------------------------------------------------------------------------
- // Estimates
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // no real estimates possible for a reducer.
- }
-
- public GroupReduceNode getCombinerUtilityNode() {
- if (this.combinerUtilityNode == null) {
- this.combinerUtilityNode = new GroupReduceNode(this);
-
- // we conservatively assume the combiner returns the same data size as it consumes
- this.combinerUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- this.combinerUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- }
- return this.combinerUtilityNode;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
deleted file mode 100644
index 1fdae51..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.util.Visitor;
-
-final class InterestingPropertiesClearer implements Visitor<OptimizerNode> {
-
- static final InterestingPropertiesClearer INSTANCE = new InterestingPropertiesClearer();
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- if (visitable.getInterestingProperties() != null) {
- visitable.clearInterestingProperties();
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
deleted file mode 100644
index 5d28043..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.util.Visitor;
-
-/**
- *
- */
-public interface IterationNode {
-
- void acceptForStepFunction(Visitor<OptimizerNode> visitor);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
deleted file mode 100644
index cbd58ca..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.operators.AbstractJoinDescriptor;
-import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
-import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.operators.SortMergeJoinDescriptor;
-import org.apache.flink.configuration.Configuration;
-
-/**
- * The Optimizer representation of a join operator.
- */
-public class JoinNode extends TwoInputNode {
-
- private List<OperatorDescriptorDual> dataProperties;
-
- /**
- * Creates a new JoinNode for the given join operator.
- *
- * @param joinOperatorBase The join operator object.
- */
- public JoinNode(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase) {
- super(joinOperatorBase);
-
- this.dataProperties = getDataProperties(joinOperatorBase,
- joinOperatorBase.getJoinHint(), joinOperatorBase.getCustomPartitioner());
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Gets the contract object for this match node.
- *
- * @return The contract.
- */
- @Override
- public JoinOperatorBase<?, ?, ?, ?> getOperator() {
- return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Join";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return this.dataProperties;
- }
-
- public void makeJoinWithSolutionSet(int solutionsetInputIndex) {
- OperatorDescriptorDual op;
- if (solutionsetInputIndex == 0) {
- op = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
- } else if (solutionsetInputIndex == 1) {
- op = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
- } else {
- throw new IllegalArgumentException();
- }
-
- this.dataProperties = Collections.singletonList(op);
- }
-
- /**
- * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
- * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
- * The result cardinality is hence the larger one.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
- long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
- this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
-
- if (this.estimatedNumRecords >= 0) {
- float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
-
- if (width > 0) {
- this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
- }
- }
- }
-
- private List<OperatorDescriptorDual> getDataProperties(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint,
- Partitioner<?> customPartitioner)
- {
- // see if an internal hint dictates the strategy to use
- Configuration conf = joinOperatorBase.getParameters();
- String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
-
- if (localStrategy != null) {
- final AbstractJoinDescriptor fixedDriverStrat;
- if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
- {
- fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
- }
- else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
- fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
- }
- else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
- fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
- }
- else {
- throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
- }
-
- if (customPartitioner != null) {
- fixedDriverStrat.setCustomPartitioner(customPartitioner);
- }
-
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
- list.add(fixedDriverStrat);
- return list;
- }
- else {
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
-
- joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint;
-
- switch (joinHint) {
- case BROADCAST_HASH_FIRST:
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, true, false, false));
- break;
- case BROADCAST_HASH_SECOND:
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, true, false));
- break;
- case REPARTITION_HASH_FIRST:
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, false, false, true));
- break;
- case REPARTITION_HASH_SECOND:
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, false, true));
- break;
- case REPARTITION_SORT_MERGE:
- list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2, false, false, true));
- break;
- case OPTIMIZER_CHOOSES:
- list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2));
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2));
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2));
- break;
- default:
- throw new CompilerException("Unrecognized join hint: " + joinHint);
- }
-
- if (customPartitioner != null) {
- for (OperatorDescriptorDual descr : list) {
- ((AbstractJoinDescriptor) descr).setCustomPartitioner(customPartitioner);
- }
- }
-
- return list;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
deleted file mode 100644
index 35def59..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.MapDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-/**
- * The optimizer's internal representation of a <i>Map</i> operator node.
- */
-public class MapNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- /**
- * Creates a new MapNode for the given operator.
- *
- * @param operator The map operator.
- */
- public MapNode(SingleInputOperator<?, ?, ?> operator) {
- super(operator);
-
- this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new MapDescriptor());
- }
-
- @Override
- public String getName() {
- return "Map";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- /**
- * Computes the estimates for the Map operator.
- * We assume that by default, Map takes one value and transforms it into another value.
- * The cardinality consequently stays the same.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
deleted file mode 100644
index b287c33..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.MapPartitionDescriptor;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-
-/**
- * The optimizer's internal representation of a <i>MapPartition</i> operator node.
- */
-public class MapPartitionNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- /**
- * Creates a new MapNode for the given contract.
- *
- * @param operator The map partition contract object.
- */
- public MapPartitionNode(SingleInputOperator<?, ?, ?> operator) {
- super(operator);
-
- this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new MapPartitionDescriptor());
- }
-
- @Override
- public String getName() {
- return "MapPartition";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- /**
- * Computes the estimates for the MapPartition operator.
- * We assume that by default, Map takes one value and transforms it into another value.
- * The cardinality consequently stays the same.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // we really cannot make any estimates here
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
deleted file mode 100644
index de3cd22..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
-import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.operators.SortMergeJoinDescriptor;
-import org.apache.flink.configuration.Configuration;
-
-/**
- * The Optimizer representation of a join operator.
- */
-public class MatchNode extends TwoInputNode {
-
- private List<OperatorDescriptorDual> dataProperties;
-
- /**
- * Creates a new MatchNode for the given join operator.
- *
- * @param joinOperatorBase The join operator object.
- */
- public MatchNode(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase) {
- super(joinOperatorBase);
- this.dataProperties = getDataProperties(joinOperatorBase, joinOperatorBase.getJoinHint());
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Gets the contract object for this match node.
- *
- * @return The contract.
- */
- @Override
- public JoinOperatorBase<?, ?, ?, ?> getOperator() {
- return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Join";
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return this.dataProperties;
- }
-
- public void makeJoinWithSolutionSet(int solutionsetInputIndex) {
- OperatorDescriptorDual op;
- if (solutionsetInputIndex == 0) {
- op = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
- } else if (solutionsetInputIndex == 1) {
- op = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
- } else {
- throw new IllegalArgumentException();
- }
-
- this.dataProperties = Collections.singletonList(op);
- }
-
- /**
- * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
- * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
- * The result cardinality is hence the larger one.
- */
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
- long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
- this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
-
- if (this.estimatedNumRecords >= 0) {
- float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
- float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
-
- if (width > 0) {
- this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
- }
- }
- }
-
- private List<OperatorDescriptorDual> getDataProperties(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint) {
- // see if an internal hint dictates the strategy to use
- Configuration conf = joinOperatorBase.getParameters();
- String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
-
- if (localStrategy != null) {
- final OperatorDescriptorDual fixedDriverStrat;
- if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
- Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
- {
- fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
- } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
- fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
- } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
- fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
- } else {
- throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
- }
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
- list.add(fixedDriverStrat);
- return list;
- }
- else {
- ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
-
- joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint;
-
- switch (joinHint) {
- case BROADCAST_HASH_FIRST:
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, true, false, false));
- break;
- case BROADCAST_HASH_SECOND:
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, true, false));
- break;
- case REPARTITION_HASH_FIRST:
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, false, false, true));
- break;
- case REPARTITION_HASH_SECOND:
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, false, true));
- break;
- case REPARTITION_SORT_MERGE:
- list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2, false, false, true));
- break;
- case OPTIMIZER_CHOOSES:
- list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2));
- list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2));
- list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2));
- break;
- default:
- throw new CompilerException("Unrecognized join hint: " + joinHint);
- }
-
- return list;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
deleted file mode 100644
index 76467cf..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.NoOpDescriptor;
-
-/**
- * The optimizer's internal representation of a <i>No Operation</i> node.
- */
-public class NoOpNode extends UnaryOperatorNode {
-
- public NoOpNode() {
- super("No Op", new FieldSet(), new NoOpDescriptor());
- }
-
- public NoOpNode(String name) {
- super(name, new FieldSet(), new NoOpDescriptor());
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- }
-}
[12/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
new file mode 100644
index 0000000..95adace
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * A node in the execution, representing a workset iteration (delta iteration).
+ */
+public class WorksetIterationPlanNode extends DualInputPlanNode implements IterationPlanNode {
+
+ private final SolutionSetPlanNode solutionSetPlanNode;
+
+ private final WorksetPlanNode worksetPlanNode;
+
+ private final PlanNode solutionSetDeltaPlanNode;
+
+ private final PlanNode nextWorkSetPlanNode;
+
+ private TypeSerializerFactory<?> worksetSerializer;
+
+ private TypeSerializerFactory<?> solutionSetSerializer;
+
+ private TypeComparatorFactory<?> solutionSetComparator;
+
+ private boolean immediateSolutionSetUpdate;
+
+ public Object postPassHelper;
+
+ private TypeSerializerFactory<?> serializerForIterationChannel;
+
+ // --------------------------------------------------------------------------------------------
+
+ public WorksetIterationPlanNode(WorksetIterationNode template, String nodeName, Channel initialSolutionSet, Channel initialWorkset,
+ SolutionSetPlanNode solutionSetPlanNode, WorksetPlanNode worksetPlanNode,
+ PlanNode nextWorkSetPlanNode, PlanNode solutionSetDeltaPlanNode)
+ {
+ super(template, nodeName, initialSolutionSet, initialWorkset, DriverStrategy.BINARY_NO_OP);
+ this.solutionSetPlanNode = solutionSetPlanNode;
+ this.worksetPlanNode = worksetPlanNode;
+ this.solutionSetDeltaPlanNode = solutionSetDeltaPlanNode;
+ this.nextWorkSetPlanNode = nextWorkSetPlanNode;
+
+ mergeBranchPlanMaps();
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public WorksetIterationNode getIterationNode() {
+ if (this.template instanceof WorksetIterationNode) {
+ return (WorksetIterationNode) this.template;
+ } else {
+ throw new RuntimeException();
+ }
+ }
+
+ public SolutionSetPlanNode getSolutionSetPlanNode() {
+ return this.solutionSetPlanNode;
+ }
+
+ public WorksetPlanNode getWorksetPlanNode() {
+ return this.worksetPlanNode;
+ }
+
+ public PlanNode getSolutionSetDeltaPlanNode() {
+ return this.solutionSetDeltaPlanNode;
+ }
+
+ public PlanNode getNextWorkSetPlanNode() {
+ return this.nextWorkSetPlanNode;
+ }
+
+ public Channel getInitialSolutionSetInput() {
+ return getInput1();
+ }
+
+ public Channel getInitialWorksetInput() {
+ return getInput2();
+ }
+
+ public void setImmediateSolutionSetUpdate(boolean immediateUpdate) {
+ this.immediateSolutionSetUpdate = immediateUpdate;
+ }
+
+ public boolean isImmediateSolutionSetUpdate() {
+ return this.immediateSolutionSetUpdate;
+ }
+
+ public FieldList getSolutionSetKeyFields() {
+ return getIterationNode().getSolutionSetKeyFields();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public TypeSerializerFactory<?> getWorksetSerializer() {
+ return worksetSerializer;
+ }
+
+ public void setWorksetSerializer(TypeSerializerFactory<?> worksetSerializer) {
+ this.worksetSerializer = worksetSerializer;
+ }
+
+ public TypeSerializerFactory<?> getSolutionSetSerializer() {
+ return solutionSetSerializer;
+ }
+
+ public void setSolutionSetSerializer(TypeSerializerFactory<?> solutionSetSerializer) {
+ this.solutionSetSerializer = solutionSetSerializer;
+ }
+
+ public TypeComparatorFactory<?> getSolutionSetComparator() {
+ return solutionSetComparator;
+ }
+
+ public void setSolutionSetComparator(TypeComparatorFactory<?> solutionSetComparator) {
+ this.solutionSetComparator = solutionSetComparator;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setCosts(Costs nodeCosts) {
+ // add the costs from the step function
+ nodeCosts.addCosts(this.solutionSetDeltaPlanNode.getCumulativeCostsShare());
+ nodeCosts.addCosts(this.nextWorkSetPlanNode.getCumulativeCostsShare());
+
+ super.setCosts(nodeCosts);
+ }
+
+ public int getMemoryConsumerWeight() {
+ // solution set index and workset back channel
+ return 2;
+ }
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+
+ SourceAndDamReport fromOutside = super.hasDamOnPathDownTo(source);
+
+ if (fromOutside == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (fromOutside == FOUND_SOURCE) {
+ // we always have a dam in the solution set index
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ SourceAndDamReport fromNextWorkset = nextWorkSetPlanNode.hasDamOnPathDownTo(source);
+
+ if (fromNextWorkset == FOUND_SOURCE_AND_DAM){
+ return FOUND_SOURCE_AND_DAM;
+ } else if (fromNextWorkset == FOUND_SOURCE){
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ return this.solutionSetDeltaPlanNode.hasDamOnPathDownTo(source);
+ }
+ }
+ }
+
+ @Override
+ public void acceptForStepFunction(Visitor<PlanNode> visitor) {
+ this.solutionSetDeltaPlanNode.accept(visitor);
+ this.nextWorkSetPlanNode.accept(visitor);
+ }
+
+ /**
+ * Merging can only take place after the solutionSetDelta and nextWorkset PlanNode has been set,
+ * because they can contain also some of the branching nodes.
+ */
+ @Override
+ protected void mergeBranchPlanMaps(Map<OptimizerNode, PlanNode> branchPlan1, Map<OptimizerNode,PlanNode> branchPlan2) {}
+
+
+ protected void mergeBranchPlanMaps() {
+ Map<OptimizerNode, PlanNode> branchPlan1 = input1.getSource().branchPlan;
+ Map<OptimizerNode, PlanNode> branchPlan2 = input2.getSource().branchPlan;
+
+ // merge the branchPlan maps according the template's uncloseBranchesStack
+ if (this.template.hasUnclosedBranches()) {
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
+ }
+
+ for (OptimizerNode.UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
+ OptimizerNode brancher = uc.getBranchingNode();
+ PlanNode selectedCandidate = null;
+
+ if (branchPlan1 != null) {
+ // predecessor 1 has branching children, see if it got the branch we are looking for
+ selectedCandidate = branchPlan1.get(brancher);
+ }
+
+ if (selectedCandidate == null && branchPlan2 != null) {
+ // predecessor 2 has branching children, see if it got the branch we are looking for
+ selectedCandidate = branchPlan2.get(brancher);
+ }
+
+ if(selectedCandidate == null && getSolutionSetDeltaPlanNode() != null && getSolutionSetDeltaPlanNode()
+ .branchPlan != null){
+ selectedCandidate = getSolutionSetDeltaPlanNode().branchPlan.get(brancher);
+ }
+
+ if(selectedCandidate == null && getNextWorkSetPlanNode() != null && getNextWorkSetPlanNode()
+ .branchPlan != null){
+ selectedCandidate = getNextWorkSetPlanNode().branchPlan.get(brancher);
+ }
+
+ if (selectedCandidate == null) {
+ throw new CompilerException(
+ "Candidates for a node with open branches are missing information about the selected candidate ");
+ }
+
+ this.branchPlan.put(brancher, selectedCandidate);
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public TypeSerializerFactory<?> getSerializerForIterationChannel() {
+ return serializerForIterationChannel;
+ }
+
+ public void setSerializerForIterationChannel(TypeSerializerFactory<?> serializerForIterationChannel) {
+ this.serializerForIterationChannel = serializerForIterationChannel;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
new file mode 100644
index 0000000..8d044d6
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.WorksetNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Plan candidate node for partial solution of a bulk iteration.
+ */
+public class WorksetPlanNode extends PlanNode {
+
+ private static final Costs NO_COSTS = new Costs();
+
+ private WorksetIterationPlanNode containingIterationNode;
+
+ private final Channel initialInput;
+
+ public Object postPassHelper;
+
+
+ public WorksetPlanNode(WorksetNode template, String nodeName,
+ GlobalProperties gProps, LocalProperties lProps,
+ Channel initialInput)
+ {
+ super(template, nodeName, DriverStrategy.NONE);
+
+ this.globalProps = gProps;
+ this.localProps = lProps;
+ this.initialInput = initialInput;
+
+ // the node incurs no cost
+ this.nodeCosts = NO_COSTS;
+ this.cumulativeCosts = NO_COSTS;
+
+ if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
+ }
+
+ this.branchPlan.putAll(initialInput.getSource().branchPlan);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public WorksetNode getWorksetNode() {
+ return (WorksetNode) this.template;
+ }
+
+ public WorksetIterationPlanNode getContainingIterationNode() {
+ return this.containingIterationNode;
+ }
+
+ public void setContainingIterationNode(WorksetIterationPlanNode containingIterationNode) {
+ this.containingIterationNode = containingIterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ return Collections.<PlanNode>emptyList();
+ }
+
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.<Channel>emptyList();
+ }
+
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+ SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
+ if (res == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (res == FOUND_SOURCE) {
+ return (this.initialInput.getLocalStrategy().dams() ||
+ this.initialInput.getTempMode().breaksPipeline() ||
+ getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
+ FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
+ }
+ else {
+ return NOT_FOUND;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
new file mode 100644
index 0000000..3f8cb46
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plandump;
+
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+
+/**
+ *
+ */
+public interface DumpableConnection<T extends DumpableNode<T>> {
+
+ public DumpableNode<T> getSource();
+
+ public ShipStrategyType getShipStrategy();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
new file mode 100644
index 0000000..1bc0f0c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plandump;
+
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.plan.PlanNode;
+
+/**
+ *
+ */
+public interface DumpableNode<T extends DumpableNode<T>> {
+
+ /**
+ * Gets an iterator over the predecessors.
+ *
+ * @return An iterator over the predecessors.
+ */
+ Iterable<T> getPredecessors();
+
+ Iterable<DumpableConnection<T>> getDumpableInputs();
+
+ OptimizerNode getOptimizerNode();
+
+ PlanNode getPlanNode();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
new file mode 100644
index 0000000..6f918c0
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
@@ -0,0 +1,657 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plandump;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.flink.api.common.operators.CompilerHints;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dag.BulkIterationNode;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.StringUtils;
+
+/**
+ *
+ */
+public class PlanJSONDumpGenerator {
+
+ private Map<DumpableNode<?>, Integer> nodeIds; // resolves pact nodes to ids
+
+ private int nodeCnt;
+
+ private boolean encodeForHTML;
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setEncodeForHTML(boolean encodeForHTML) {
+ this.encodeForHTML = encodeForHTML;
+ }
+
+ public boolean isEncodeForHTML() {
+ return encodeForHTML;
+ }
+
+
+ public void dumpPactPlanAsJSON(List<DataSinkNode> nodes, PrintWriter writer) {
+ @SuppressWarnings("unchecked")
+ List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
+ compilePlanToJSON(n, writer);
+ }
+
+ public String getPactPlanAsJSON(List<DataSinkNode> nodes) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ dumpPactPlanAsJSON(nodes, pw);
+ return sw.toString();
+ }
+
+ public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, File toFile) throws IOException {
+ PrintWriter pw = null;
+ try {
+ pw = new PrintWriter(new FileOutputStream(toFile), false);
+ dumpOptimizerPlanAsJSON(plan, pw);
+ pw.flush();
+ } finally {
+ if (pw != null) {
+ pw.close();
+ }
+ }
+ }
+
+ public String getOptimizerPlanAsJSON(OptimizedPlan plan) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ dumpOptimizerPlanAsJSON(plan, pw);
+ pw.close();
+ return sw.toString();
+ }
+
+ public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) {
+ Collection<SinkPlanNode> sinks = plan.getDataSinks();
+ if (sinks instanceof List) {
+ dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer);
+ } else {
+ List<SinkPlanNode> n = new ArrayList<SinkPlanNode>();
+ n.addAll(sinks);
+ dumpOptimizerPlanAsJSON(n, writer);
+ }
+ }
+
+ public void dumpOptimizerPlanAsJSON(List<SinkPlanNode> nodes, PrintWriter writer) {
+ @SuppressWarnings("unchecked")
+ List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
+ compilePlanToJSON(n, writer);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private void compilePlanToJSON(List<DumpableNode<?>> nodes, PrintWriter writer) {
+ // initialization to assign node ids
+ this.nodeIds = new HashMap<DumpableNode<?>, Integer>();
+ this.nodeCnt = 0;
+
+ // JSON header
+ writer.print("{\n\t\"nodes\": [\n\n");
+
+ // Generate JSON for plan
+ for (int i = 0; i < nodes.size(); i++) {
+ visit(nodes.get(i), writer, i == 0);
+ }
+
+ // JSON Footer
+ writer.println("\n\t]\n}");
+ }
+
+ private boolean visit(DumpableNode<?> node, PrintWriter writer, boolean first) {
+ // check for duplicate traversal
+ if (this.nodeIds.containsKey(node)) {
+ return false;
+ }
+
+ // assign an id first
+ this.nodeIds.put(node, this.nodeCnt++);
+
+ // then recurse
+ for (DumpableNode<?> child : node.getPredecessors()) {
+ //This is important, because when the node was already in the graph it is not allowed
+ //to set first to false!
+ if (visit(child, writer, first)) {
+ first = false;
+ };
+ }
+
+ // check if this node should be skipped from the dump
+ final OptimizerNode n = node.getOptimizerNode();
+
+ // ------------------ dump after the ascend ---------------------
+ // start a new node and output node id
+ if (!first) {
+ writer.print(",\n");
+ }
+ // open the node
+ writer.print("\t{\n");
+
+ // recurse, it is is an iteration node
+ if (node instanceof BulkIterationNode || node instanceof BulkIterationPlanNode) {
+
+ DumpableNode<?> innerChild = node instanceof BulkIterationNode ?
+ ((BulkIterationNode) node).getNextPartialSolution() :
+ ((BulkIterationPlanNode) node).getRootOfStepFunction();
+
+ DumpableNode<?> begin = node instanceof BulkIterationNode ?
+ ((BulkIterationNode) node).getPartialSolution() :
+ ((BulkIterationPlanNode) node).getPartialSolutionPlanNode();
+
+ writer.print("\t\t\"step_function\": [\n");
+
+ visit(innerChild, writer, true);
+
+ writer.print("\n\t\t],\n");
+ writer.print("\t\t\"partial_solution\": " + this.nodeIds.get(begin) + ",\n");
+ writer.print("\t\t\"next_partial_solution\": " + this.nodeIds.get(innerChild) + ",\n");
+ } else if (node instanceof WorksetIterationNode || node instanceof WorksetIterationPlanNode) {
+
+ DumpableNode<?> worksetRoot = node instanceof WorksetIterationNode ?
+ ((WorksetIterationNode) node).getNextWorkset() :
+ ((WorksetIterationPlanNode) node).getNextWorkSetPlanNode();
+ DumpableNode<?> solutionDelta = node instanceof WorksetIterationNode ?
+ ((WorksetIterationNode) node).getSolutionSetDelta() :
+ ((WorksetIterationPlanNode) node).getSolutionSetDeltaPlanNode();
+
+ DumpableNode<?> workset = node instanceof WorksetIterationNode ?
+ ((WorksetIterationNode) node).getWorksetNode() :
+ ((WorksetIterationPlanNode) node).getWorksetPlanNode();
+ DumpableNode<?> solutionSet = node instanceof WorksetIterationNode ?
+ ((WorksetIterationNode) node).getSolutionSetNode() :
+ ((WorksetIterationPlanNode) node).getSolutionSetPlanNode();
+
+ writer.print("\t\t\"step_function\": [\n");
+
+ visit(worksetRoot, writer, true);
+ visit(solutionDelta, writer, false);
+
+ writer.print("\n\t\t],\n");
+ writer.print("\t\t\"workset\": " + this.nodeIds.get(workset) + ",\n");
+ writer.print("\t\t\"solution_set\": " + this.nodeIds.get(solutionSet) + ",\n");
+ writer.print("\t\t\"next_workset\": " + this.nodeIds.get(worksetRoot) + ",\n");
+ writer.print("\t\t\"solution_delta\": " + this.nodeIds.get(solutionDelta) + ",\n");
+ }
+
+ // print the id
+ writer.print("\t\t\"id\": " + this.nodeIds.get(node));
+
+
+ final String type;
+ String contents;
+ if (n instanceof DataSinkNode) {
+ type = "sink";
+ contents = n.getOperator().toString();
+ } else if (n instanceof DataSourceNode) {
+ type = "source";
+ contents = n.getOperator().toString();
+ }
+ else if (n instanceof BulkIterationNode) {
+ type = "bulk_iteration";
+ contents = n.getOperator().getName();
+ }
+ else if (n instanceof WorksetIterationNode) {
+ type = "workset_iteration";
+ contents = n.getOperator().getName();
+ }
+ else if (n instanceof BinaryUnionNode) {
+ type = "pact";
+ contents = "";
+ }
+ else {
+ type = "pact";
+ contents = n.getOperator().getName();
+ }
+
+ contents = StringUtils.showControlCharacters(contents);
+ if (encodeForHTML) {
+ contents = StringEscapeUtils.escapeHtml4(contents);
+ contents = contents.replace("\\", "\");
+ }
+
+
+ String name = n.getName();
+ if (name.equals("Reduce") && (node instanceof SingleInputPlanNode) &&
+ ((SingleInputPlanNode) node).getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE) {
+ name = "Combine";
+ }
+
+ // output the type identifier
+ writer.print(",\n\t\t\"type\": \"" + type + "\"");
+
+ // output node name
+ writer.print(",\n\t\t\"pact\": \"" + name + "\"");
+
+ // output node contents
+ writer.print(",\n\t\t\"contents\": \"" + contents + "\"");
+
+ // degree of parallelism
+ writer.print(",\n\t\t\"parallelism\": \""
+ + (n.getParallelism() >= 1 ? n.getParallelism() : "default") + "\"");
+
+ // output node predecessors
+ Iterator<? extends DumpableConnection<?>> inConns = node.getDumpableInputs().iterator();
+ String child1name = "", child2name = "";
+
+ if (inConns != null && inConns.hasNext()) {
+ // start predecessor list
+ writer.print(",\n\t\t\"predecessors\": [");
+ int inputNum = 0;
+
+ while (inConns.hasNext()) {
+ final DumpableConnection<?> inConn = inConns.next();
+ final DumpableNode<?> source = inConn.getSource();
+ writer.print(inputNum == 0 ? "\n" : ",\n");
+ if (inputNum == 0) {
+ child1name += child1name.length() > 0 ? ", " : "";
+ child1name += source.getOptimizerNode().getOperator().getName();
+ } else if (inputNum == 1) {
+ child2name += child2name.length() > 0 ? ", " : "";
+ child2name = source.getOptimizerNode().getOperator().getName();
+ }
+
+ // output predecessor id
+ writer.print("\t\t\t{\"id\": " + this.nodeIds.get(source));
+
+ // output connection side
+ if (inConns.hasNext() || inputNum > 0) {
+ writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
+ }
+ // output shipping strategy and channel type
+ final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
+ final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
+ ((DagConnection) inConn).getShipStrategy();
+
+ String shipStrategy = null;
+ if (shipType != null) {
+ switch (shipType) {
+ case NONE:
+ // nothing
+ break;
+ case FORWARD:
+ shipStrategy = "Forward";
+ break;
+ case BROADCAST:
+ shipStrategy = "Broadcast";
+ break;
+ case PARTITION_HASH:
+ shipStrategy = "Hash Partition";
+ break;
+ case PARTITION_RANGE:
+ shipStrategy = "Range Partition";
+ break;
+ case PARTITION_RANDOM:
+ shipStrategy = "Redistribute";
+ break;
+ case PARTITION_FORCED_REBALANCE:
+ shipStrategy = "Rebalance";
+ break;
+ case PARTITION_CUSTOM:
+ shipStrategy = "Custom Partition";
+ break;
+ default:
+ throw new CompilerException("Unknown ship strategy '" + inConn.getShipStrategy().name()
+ + "' in JSON generator.");
+ }
+ }
+
+ if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
+ shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ?
+ channel.getShipStrategyKeys().toString() :
+ Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
+ }
+
+ if (shipStrategy != null) {
+ writer.print(", \"ship_strategy\": \"" + shipStrategy + "\"");
+ }
+
+ if (channel != null) {
+ String localStrategy = null;
+ switch (channel.getLocalStrategy()) {
+ case NONE:
+ break;
+ case SORT:
+ localStrategy = "Sort";
+ break;
+ case COMBININGSORT:
+ localStrategy = "Sort (combining)";
+ break;
+ default:
+ throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name());
+ }
+
+ if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
+ localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ?
+ channel.getLocalStrategyKeys().toString() :
+ Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
+ }
+
+ if (localStrategy != null) {
+ writer.print(", \"local_strategy\": \"" + localStrategy + "\"");
+ }
+
+ if (channel != null && channel.getTempMode() != TempMode.NONE) {
+ String tempMode = channel.getTempMode().toString();
+ writer.print(", \"temp_mode\": \"" + tempMode + "\"");
+ }
+ }
+
+ writer.print('}');
+ inputNum++;
+ }
+ // finish predecessors
+ writer.print("\n\t\t]");
+ }
+
+ //---------------------------------------------------------------------------------------
+ // the part below here is relevant only to plan nodes with concrete strategies, etc
+ //---------------------------------------------------------------------------------------
+
+ final PlanNode p = node.getPlanNode();
+ if (p == null) {
+ // finish node
+ writer.print("\n\t}");
+ return true;
+ }
+ // local strategy
+ String locString = null;
+ if (p.getDriverStrategy() != null) {
+ switch (p.getDriverStrategy()) {
+ case NONE:
+ case BINARY_NO_OP:
+ break;
+
+ case UNARY_NO_OP:
+ locString = "No-Op";
+ break;
+
+ case COLLECTOR_MAP:
+ case MAP:
+ locString = "Map";
+ break;
+
+ case FLAT_MAP:
+ locString = "FlatMap";
+ break;
+
+ case MAP_PARTITION:
+ locString = "Map Partition";
+ break;
+
+ case ALL_REDUCE:
+ locString = "Reduce All";
+ break;
+
+ case ALL_GROUP_REDUCE:
+ case ALL_GROUP_REDUCE_COMBINE:
+ locString = "Group Reduce All";
+ break;
+
+ case SORTED_REDUCE:
+ locString = "Sorted Reduce";
+ break;
+
+ case SORTED_PARTIAL_REDUCE:
+ locString = "Sorted Combine/Reduce";
+ break;
+
+ case SORTED_GROUP_REDUCE:
+ locString = "Sorted Group Reduce";
+ break;
+
+ case SORTED_GROUP_COMBINE:
+ locString = "Sorted Combine";
+ break;
+
+ case HYBRIDHASH_BUILD_FIRST:
+ locString = "Hybrid Hash (build: " + child1name + ")";
+ break;
+ case HYBRIDHASH_BUILD_SECOND:
+ locString = "Hybrid Hash (build: " + child2name + ")";
+ break;
+
+ case HYBRIDHASH_BUILD_FIRST_CACHED:
+ locString = "Hybrid Hash (CACHED) (build: " + child1name + ")";
+ break;
+ case HYBRIDHASH_BUILD_SECOND_CACHED:
+ locString = "Hybrid Hash (CACHED) (build: " + child2name + ")";
+ break;
+
+ case NESTEDLOOP_BLOCKED_OUTER_FIRST:
+ locString = "Nested Loops (Blocked Outer: " + child1name + ")";
+ break;
+ case NESTEDLOOP_BLOCKED_OUTER_SECOND:
+ locString = "Nested Loops (Blocked Outer: " + child2name + ")";
+ break;
+ case NESTEDLOOP_STREAMED_OUTER_FIRST:
+ locString = "Nested Loops (Streamed Outer: " + child1name + ")";
+ break;
+ case NESTEDLOOP_STREAMED_OUTER_SECOND:
+ locString = "Nested Loops (Streamed Outer: " + child2name + ")";
+ break;
+
+ case MERGE:
+ locString = "Merge";
+ break;
+
+ case CO_GROUP:
+ locString = "Co-Group";
+ break;
+
+ default:
+ locString = p.getDriverStrategy().name();
+ break;
+ }
+
+ if (locString != null) {
+ writer.print(",\n\t\t\"driver_strategy\": \"");
+ writer.print(locString);
+ writer.print("\"");
+ }
+ }
+
+ {
+ // output node global properties
+ final GlobalProperties gp = p.getGlobalProperties();
+
+ writer.print(",\n\t\t\"global_properties\": [\n");
+
+ addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
+ if (gp.getPartitioningFields() != null) {
+ addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
+ }
+ if (gp.getPartitioningOrdering() != null) {
+ addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false);
+ }
+ else {
+ addProperty(writer, "Partitioning Order", "(none)", false);
+ }
+ if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
+ addProperty(writer, "Uniqueness", "not unique", false);
+ }
+ else {
+ addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);
+ }
+
+ writer.print("\n\t\t]");
+ }
+
+ {
+ // output node local properties
+ LocalProperties lp = p.getLocalProperties();
+
+ writer.print(",\n\t\t\"local_properties\": [\n");
+
+ if (lp.getOrdering() != null) {
+ addProperty(writer, "Order", lp.getOrdering().toString(), true);
+ }
+ else {
+ addProperty(writer, "Order", "(none)", true);
+ }
+ if (lp.getGroupedFields() != null && lp.getGroupedFields().size() > 0) {
+ addProperty(writer, "Grouped on", lp.getGroupedFields().toString(), false);
+ } else {
+ addProperty(writer, "Grouping", "not grouped", false);
+ }
+ if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
+ addProperty(writer, "Uniqueness", "not unique", false);
+ }
+ else {
+ addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);
+ }
+
+ writer.print("\n\t\t]");
+ }
+
+ // output node size estimates
+ writer.print(",\n\t\t\"estimates\": [\n");
+
+ addProperty(writer, "Est. Output Size", n.getEstimatedOutputSize() == -1 ? "(unknown)"
+ : formatNumber(n.getEstimatedOutputSize(), "B"), true);
+ addProperty(writer, "Est. Cardinality", n.getEstimatedNumRecords() == -1 ? "(unknown)"
+ : formatNumber(n.getEstimatedNumRecords()), false);
+
+ writer.print("\t\t]");
+
+ // output node cost
+ if (p.getNodeCosts() != null) {
+ writer.print(",\n\t\t\"costs\": [\n");
+
+ addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)"
+ : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true);
+ addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)"
+ : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false);
+ addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)"
+ : formatNumber(p.getNodeCosts().getCpuCost(), ""), false);
+
+ addProperty(writer, "Cumulative Network",
+ p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p
+ .getCumulativeCosts().getNetworkCost(), "B"), false);
+ addProperty(writer, "Cumulative Disk I/O",
+ p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p
+ .getCumulativeCosts().getDiskCost(), "B"), false);
+ addProperty(writer, "Cumulative CPU",
+ p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
+ .getCumulativeCosts().getCpuCost(), ""), false);
+
+ writer.print("\n\t\t]");
+ }
+
+ // output the node compiler hints
+ if (n.getOperator().getCompilerHints() != null) {
+ CompilerHints hints = n.getOperator().getCompilerHints();
+ CompilerHints defaults = new CompilerHints();
+
+ String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize());
+ String card = hints.getOutputCardinality() == defaults.getOutputCardinality() ? "(none)" : String.valueOf(hints.getOutputCardinality());
+ String width = hints.getAvgOutputRecordSize() == defaults.getAvgOutputRecordSize() ? "(none)" : String.valueOf(hints.getAvgOutputRecordSize());
+ String filter = hints.getFilterFactor() == defaults.getFilterFactor() ? "(none)" : String.valueOf(hints.getFilterFactor());
+
+ writer.print(",\n\t\t\"compiler_hints\": [\n");
+
+ addProperty(writer, "Output Size (bytes)", size, true);
+ addProperty(writer, "Output Cardinality", card, false);
+ addProperty(writer, "Avg. Output Record Size (bytes)", width, false);
+ addProperty(writer, "Filter Factor", filter, false);
+
+ writer.print("\t\t]");
+ }
+
+ // finish node
+ writer.print("\n\t}");
+ return true;
+ }
+
+ private void addProperty(PrintWriter writer, String name, String value, boolean first) {
+ if (!first) {
+ writer.print(",\n");
+ }
+ writer.print("\t\t\t{ \"name\": \"");
+ writer.print(name);
+ writer.print("\", \"value\": \"");
+ writer.print(value);
+ writer.print("\" }");
+ }
+
+ public static final String formatNumber(double number) {
+ return formatNumber(number, "");
+ }
+
+ public static final String formatNumber(double number, String suffix) {
+ if (number <= 0.0) {
+ return String.valueOf(number);
+ }
+
+ int power = (int) Math.ceil(Math.log10(number));
+
+ int group = (power - 1) / 3;
+ if (group >= SIZE_SUFFIXES.length) {
+ group = SIZE_SUFFIXES.length - 1;
+ } else if (group < 0) {
+ group = 0;
+ }
+
+ // truncate fractional part
+ int beforeDecimal = power - group * 3;
+ if (power > beforeDecimal) {
+ for (int i = power - beforeDecimal; i > 0; i--) {
+ number /= 10;
+ }
+ }
+
+ return group > 0 ? String.format(Locale.US, "%.2f %s", number, SIZE_SUFFIXES[group]) :
+ String.format(Locale.US, "%.2f", number);
+ }
+
+ private static final char[] SIZE_SUFFIXES = { 0, 'K', 'M', 'G', 'T' };
+}
[05/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
new file mode 100644
index 0000000..f6885c5
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.junit.Test;
+
+/**
+ * This test case has been created to validate a bug that occurred when
+ * the ReduceOperator was used without a grouping key.
+ */
+@SuppressWarnings({"serial", "deprecation"})
+public class ReduceAllTest extends CompilerTestBase {
+
+ @Test
+ public void testReduce() {
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce()).name("Reduce1").input(source).build();
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setInput(reduce1);
+ Plan plan = new Plan(sink, "AllReduce Test");
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+
+ try {
+ OptimizedPlan oPlan = compileNoStats(plan);
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+ jobGen.compileJobGraph(oPlan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
new file mode 100644
index 0000000..da44b59
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.MapPartitionFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.io.ReplicatingInputFormat;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.CsvInputFormat;
+import org.apache.flink.api.java.operators.DataSink;
+import org.apache.flink.api.java.operators.translation.JavaPlan;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.core.fs.FileInputSplit;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+import org.junit.Assert;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "deprecation"})
+public class ReplicatingDataSourceTest extends CompilerTestBase {
+
+ /**
+ * Tests join program with replicated data source.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInput() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests join program with replicated data source behind map.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInputBehindMap() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .map(new IdMap())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests join program with replicated data source behind filter.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInputBehindFilter() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .filter(new NoFilter())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests join program with replicated data source behind flatMap.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInputBehindFlatMap() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .flatMap(new IdFlatMap())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests join program with replicated data source behind map partition.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInputBehindMapPartition() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .mapPartition(new IdPMap())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests join program with replicated data source behind multiple map ops.
+ */
+ @Test
+ public void checkJoinWithReplicatedSourceInputBehindMultiMaps() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .filter(new NoFilter())
+ .mapPartition(new IdPMap())
+ .flatMap(new IdFlatMap())
+ .map(new IdMap())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when join should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
+ ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
+ }
+
+ /**
+ * Tests cross program with replicated data source.
+ */
+ @Test
+ public void checkCrossWithReplicatedSourceInput() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .cross(source2)
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when cross should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
+ ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
+ }
+
+ /**
+ * Tests cross program with replicated data source behind map and filter.
+ */
+ @Test
+ public void checkCrossWithReplicatedSourceInputBehindMap() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .map(new IdMap())
+ .filter(new NoFilter())
+ .cross(source2)
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ // when cross should have forward strategy on both sides
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
+ ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();
+
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
+ Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
+ }
+
+ /**
+ * Tests compiler fail for join program with replicated data source and changing DOP.
+ */
+ @Test(expected = CompilerException.class)
+ public void checkJoinWithReplicatedSourceInputChangingDOP() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .join(source2).where("*").equalTo("*").setParallelism(DEFAULT_PARALLELISM+2)
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+ }
+
+ /**
+ * Tests compiler fail for join program with replicated data source behind map and changing DOP.
+ */
+ @Test(expected = CompilerException.class)
+ public void checkJoinWithReplicatedSourceInputBehindMapChangingDOP() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .map(new IdMap()).setParallelism(DEFAULT_PARALLELISM+1)
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ }
+
+ /**
+ * Tests compiler fail for join program with replicated data source behind reduce.
+ */
+ @Test(expected = CompilerException.class)
+ public void checkJoinWithReplicatedSourceInputBehindReduce() {
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .reduce(new LastReduce())
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+ }
+
+ /**
+ * Tests compiler fail for join program with replicated data source behind rebalance.
+ */
+ @Test(expected = CompilerException.class)
+ public void checkJoinWithReplicatedSourceInputBehindRebalance() {
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
+ new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
+
+ DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
+ DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
+
+ DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
+ .rebalance()
+ .join(source2).where("*").equalTo("*")
+ .writeAsText("/some/newpath");
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+ }
+
+
+ public static class IdMap<T> implements MapFunction<T,T> {
+
+ @Override
+ public T map(T value) throws Exception {
+ return value;
+ }
+ }
+
+ public static class NoFilter<T> implements FilterFunction<T> {
+
+ @Override
+ public boolean filter(T value) throws Exception {
+ return false;
+ }
+ }
+
+ public static class IdFlatMap<T> implements FlatMapFunction<T,T> {
+
+ @Override
+ public void flatMap(T value, Collector<T> out) throws Exception {
+ out.collect(value);
+ }
+ }
+
+ public static class IdPMap<T> implements MapPartitionFunction<T,T> {
+
+ @Override
+ public void mapPartition(Iterable<T> values, Collector<T> out) throws Exception {
+ for(T v : values) {
+ out.collect(v);
+ }
+ }
+ }
+
+ public static class LastReduce<T> implements ReduceFunction<T> {
+
+ @Override
+ public T reduce(T value1, T value2) throws Exception {
+ return value2;
+ }
+ }
+
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
new file mode 100644
index 0000000..1fe16bb
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.translation.JavaPlan;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+import org.junit.Assert;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class SemanticPropertiesAPIToPlanTest extends CompilerTestBase {
+
+ @Test
+ public void forwardFieldsTestMapReduce() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ set = set.map(new MockMapper()).withForwardedFields("*")
+ .groupBy(0)
+ .reduce(new MockReducer()).withForwardedFields("f0->f1")
+ .map(new MockMapper()).withForwardedFields("*")
+ .groupBy(1)
+ .reduce(new MockReducer()).withForwardedFields("*");
+
+ set.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ oPlan.accept(new Visitor<PlanNode>() {
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperatorBase) {
+ for (Channel input: visitable.getInputs()) {
+ GlobalProperties gprops = visitable.getGlobalProperties();
+ LocalProperties lprops = visitable.getLocalProperties();
+
+ Assert.assertTrue("Reduce should just forward the input if it is already partitioned",
+ input.getShipStrategy() == ShipStrategyType.FORWARD);
+ Assert.assertTrue("Wrong GlobalProperties on Reducer",
+ gprops.isPartitionedOnFields(new FieldSet(1)));
+ Assert.assertTrue("Wrong GlobalProperties on Reducer",
+ gprops.getPartitioning() == PartitioningProperty.HASH_PARTITIONED);
+ Assert.assertTrue("Wrong LocalProperties on Reducer",
+ lprops.getGroupedFields().contains(1));
+ }
+ }
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof MapOperatorBase) {
+ for (Channel input: visitable.getInputs()) {
+ GlobalProperties gprops = visitable.getGlobalProperties();
+ LocalProperties lprops = visitable.getLocalProperties();
+
+ Assert.assertTrue("Map should just forward the input if it is already partitioned",
+ input.getShipStrategy() == ShipStrategyType.FORWARD);
+ Assert.assertTrue("Wrong GlobalProperties on Mapper",
+ gprops.isPartitionedOnFields(new FieldSet(1)));
+ Assert.assertTrue("Wrong GlobalProperties on Mapper",
+ gprops.getPartitioning() == PartitioningProperty.HASH_PARTITIONED);
+ Assert.assertTrue("Wrong LocalProperties on Mapper",
+ lprops.getGroupedFields().contains(1));
+ }
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+
+ }
+ });
+ }
+
+ @Test
+ public void forwardFieldsTestJoin() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> in1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> in2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ in1 = in1.map(new MockMapper()).withForwardedFields("*")
+ .groupBy(0)
+ .reduce(new MockReducer()).withForwardedFields("f0->f1");
+ in2 = in2.map(new MockMapper()).withForwardedFields("*")
+ .groupBy(1)
+ .reduce(new MockReducer()).withForwardedFields("f1->f2");
+ DataSet<Tuple3<Integer, Integer, Integer>> out = in1.join(in2).where(1).equalTo(2).with(new MockJoin());
+
+ out.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ oPlan.accept(new Visitor<PlanNode>() {
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (visitable instanceof DualInputPlanNode && visitable.getProgramOperator() instanceof JoinOperatorBase) {
+ DualInputPlanNode node = ((DualInputPlanNode) visitable);
+
+ final Channel inConn1 = node.getInput1();
+ final Channel inConn2 = node.getInput2();
+
+ Assert.assertTrue("Join should just forward the input if it is already partitioned",
+ inConn1.getShipStrategy() == ShipStrategyType.FORWARD);
+ Assert.assertTrue("Join should just forward the input if it is already partitioned",
+ inConn2.getShipStrategy() == ShipStrategyType.FORWARD);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+
+ }
+ });
+ }
+
+ public static class MockMapper implements MapFunction<Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+ @Override
+ public Tuple3<Integer, Integer, Integer> map(Tuple3<Integer, Integer, Integer> value) throws Exception {
+ return null;
+ }
+ }
+
+ public static class MockReducer implements ReduceFunction<Tuple3<Integer, Integer, Integer>> {
+ @Override
+ public Tuple3<Integer, Integer, Integer> reduce(Tuple3<Integer, Integer, Integer> value1, Tuple3<Integer, Integer, Integer> value2) throws Exception {
+ return null;
+ }
+ }
+
+ public static class MockJoin implements JoinFunction<Tuple3<Integer, Integer, Integer>,
+ Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+ @Override
+ public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
+ return null;
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
new file mode 100644
index 0000000..40b54e0
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+
+@SuppressWarnings("serial")
+public class SortPartialReuseTest extends CompilerTestBase {
+
+ @Test
+ public void testPartialPartitioningReuse() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input
+ .partitionByHash(0)
+ .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
+
+ .groupBy(0, 1)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
+
+ .groupBy(0)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
+
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer2 = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode reducer1 = (SingleInputPlanNode) reducer2.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+
+ // should be locally forwarding, reusing sort and partitioning
+ assertEquals(ShipStrategyType.FORWARD, reducer2.getInput().getShipStrategy());
+ assertEquals(LocalStrategy.NONE, reducer2.getInput().getLocalStrategy());
+
+ assertEquals(ShipStrategyType.FORWARD, reducer1.getInput().getShipStrategy());
+ assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCustomPartitioningNotReused() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input
+ .partitionCustom(new Partitioner<Long>() {
+ @Override
+ public int partition(Long key, int numPartitions) { return 0; }
+ }, 0)
+ .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
+
+ .groupBy(0, 1)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
+
+ .groupBy(1)
+ .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
+
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer2 = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer2.getInput().getSource();
+ SingleInputPlanNode reducer1 = (SingleInputPlanNode) combiner.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+
+ // should be locally forwarding, reusing sort and partitioning
+ assertEquals(ShipStrategyType.PARTITION_HASH, reducer2.getInput().getShipStrategy());
+ assertEquals(LocalStrategy.COMBININGSORT, reducer2.getInput().getLocalStrategy());
+
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+ assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
+
+ assertEquals(ShipStrategyType.FORWARD, reducer1.getInput().getShipStrategy());
+ assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
new file mode 100644
index 0000000..92b4fc5
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class UnionBetweenDynamicAndStaticPathTest extends CompilerTestBase {
+
+ @Test
+ public void testUnionStaticFirst() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> input1 = env.generateSequence(1, 10);
+ DataSet<Long> input2 = env.generateSequence(1, 10);
+
+ IterativeDataSet<Long> iteration = input1.iterate(10);
+
+ DataSet<Long> result = iteration.closeWith(
+ input2.union(input2).union(iteration.union(iteration)));
+
+ result.print();
+ result.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(2, op.getDataSinks().size());
+
+ BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
+ BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
+ NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
+ NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
+
+ assertTrue(mixedUnion.unionsStaticAndDynamicPath());
+ assertFalse(mixedUnion.getInput1().isOnDynamicPath());
+ assertTrue(mixedUnion.getInput2().isOnDynamicPath());
+ assertTrue(mixedUnion.getInput1().getTempMode().isCached());
+
+ for (Channel c : staticUnion.getInputs()) {
+ assertFalse(c.isOnDynamicPath());
+ }
+ for (Channel c : dynamicUnion.getInputs()) {
+ assertTrue(c.isOnDynamicPath());
+ }
+
+ assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
+ assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
+ assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testUnionStaticSecond() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> input1 = env.generateSequence(1, 10);
+ DataSet<Long> input2 = env.generateSequence(1, 10);
+
+ IterativeDataSet<Long> iteration = input1.iterate(10);
+
+ DataSet<Long> iterResult = iteration
+ .closeWith(iteration.union(iteration).union(input2.union(input2)));
+
+ iterResult.print();
+ iterResult.print();
+
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(2, op.getDataSinks().size());
+
+ BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
+ BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
+ NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
+ NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
+
+ assertTrue(mixedUnion.unionsStaticAndDynamicPath());
+ assertFalse(mixedUnion.getInput1().isOnDynamicPath());
+ assertTrue(mixedUnion.getInput2().isOnDynamicPath());
+ assertTrue(mixedUnion.getInput1().getTempMode().isCached());
+
+ assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
+ assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
+ assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
+
+ for (Channel c : staticUnion.getInputs()) {
+ assertFalse(c.isOnDynamicPath());
+ }
+ for (Channel c : dynamicUnion.getInputs()) {
+ assertTrue(c.isOnDynamicPath());
+ }
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
new file mode 100644
index 0000000..5d15ed8
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer;
+
+import java.util.Iterator;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.java.aggregation.Aggregations;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.Visitor;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityReduce;
+
+
+@SuppressWarnings({"serial", "deprecation"})
+public class UnionPropertyPropagationTest extends CompilerTestBase {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testUnionPropertyOldApiPropagation() {
+ // construct the plan
+
+ FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
+ FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
+
+ ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(sourceA)
+ .build();
+ ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(sourceB)
+ .build();
+
+ ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
+ globalRed.addInput(redA);
+ globalRed.addInput(redB);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
+
+ // return the plan
+ Plan plan = new Plan(sink, "Union Property Propagation");
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+
+ // Compile plan to verify that no error is thrown
+ jobGen.compileJobGraph(oPlan);
+
+ oPlan.accept(new Visitor<PlanNode>() {
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperator) {
+ for (Channel inConn : visitable.getInputs()) {
+ Assert.assertTrue("Reduce should just forward the input if it is already partitioned",
+ inConn.getShipStrategy() == ShipStrategyType.FORWARD);
+ }
+ //just check latest ReduceNode
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+ // DO NOTHING
+ }
+ });
+ }
+
+ @Test
+ public void testUnionNewApiAssembly() {
+ final int NUM_INPUTS = 4;
+
+ // construct the plan it will be multiple flat maps, all unioned
+ // and the "unioned" dataSet will be grouped
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> source = env.readTextFile(IN_FILE);
+ DataSet<Tuple2<String, Integer>> lastUnion = source.flatMap(new DummyFlatMap());
+
+ for (int i = 1; i< NUM_INPUTS; i++){
+ lastUnion = lastUnion.union(source.flatMap(new DummyFlatMap()));
+ }
+
+ DataSet<Tuple2<String, Integer>> result = lastUnion.groupBy(0).aggregate(Aggregations.SUM, 1);
+ result.writeAsText(OUT_FILE);
+
+ // return the plan
+ Plan plan = env.createProgramPlan("Test union on new java-api");
+ OptimizedPlan oPlan = compileNoStats(plan);
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+
+ // Compile plan to verify that no error is thrown
+ jobGen.compileJobGraph(oPlan);
+
+ oPlan.accept(new Visitor<PlanNode>() {
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+
+ /* Test on the union output connections
+ * It must be under the GroupOperator and the strategy should be forward
+ */
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof GroupReduceOperatorBase){
+ final Channel inConn = ((SingleInputPlanNode) visitable).getInput();
+ Assert.assertTrue("Union should just forward the Partitioning",
+ inConn.getShipStrategy() == ShipStrategyType.FORWARD );
+ Assert.assertTrue("Union Node should be under Group operator",
+ inConn.getSource() instanceof NAryUnionPlanNode );
+ }
+
+ /* Test on the union input connections
+ * Must be NUM_INPUTS input connections, all FlatMapOperators with a own partitioning strategy(propably hash)
+ */
+ if (visitable instanceof NAryUnionPlanNode) {
+ int numberInputs = 0;
+ for (Iterator<Channel> inputs = visitable.getInputs().iterator(); inputs.hasNext(); numberInputs++) {
+ final Channel inConn = inputs.next();
+ PlanNode inNode = inConn.getSource();
+ Assert.assertTrue("Input of Union should be FlatMapOperators",
+ inNode.getProgramOperator() instanceof FlatMapOperatorBase);
+ Assert.assertTrue("Shipment strategy under union should partition the data",
+ inConn.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
+ }
+
+ Assert.assertTrue("NAryUnion should have " + NUM_INPUTS + " inputs", numberInputs == NUM_INPUTS);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+ // DO NOTHING
+ }
+ });
+ }
+
+ public static final class DummyFlatMap extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+ out.collect(new Tuple2<String, Integer>(value, 0));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
new file mode 100644
index 0000000..1e4124c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+@SuppressWarnings("serial")
+public class UnionReplacementTest extends CompilerTestBase {
+
+ @Test
+ public void testUnionReplacement() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<String> input1 = env.fromElements("test1");
+ DataSet<String> input2 = env.fromElements("test2");
+
+ DataSet<String> union = input1.union(input2);
+
+ union.print();
+ union.print();
+
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileNoStats(plan);
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+ jobGen.compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
new file mode 100644
index 0000000..80c0bda
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class WorksetIterationCornerCasesTest extends CompilerTestBase {
+
+ @Test
+ public void testWorksetIterationNotDependingOnSolutionSet() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 100).map(new Duplicator<Long>());
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 100, 1);
+
+ DataSet<Tuple2<Long, Long>> iterEnd = iteration.getWorkset().map(new TestMapper<Tuple2<Long,Long>>());
+ iteration.closeWith(iterEnd, iterEnd).print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+ assertTrue(wipn.getSolutionSetPlanNode().getOutgoingChannels().isEmpty());
+
+ JobGraphGenerator jgg = new JobGraphGenerator();
+ jgg.compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static final class Duplicator<T> implements MapFunction<T, Tuple2<T, T>> {
+ @Override
+ public Tuple2<T, T> map(T value) {
+ return new Tuple2<T, T>(value, value);
+ }
+ }
+
+ private static final class TestMapper<T> implements MapFunction<T, T> {
+ @Override
+ public T map(T value) {
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
new file mode 100644
index 0000000..6e7c0a3
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.record.operators.DeltaIteration;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.JoinOperator;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyMatchStub;
+import org.apache.flink.optimizer.util.DummyNonPreservingMatchStub;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityMap;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.types.LongValue;
+import org.junit.Test;
+
+
+/**
+* Tests that validate optimizer choices when using operators that are requesting certain specific execution
+* strategies.
+*/
+@SuppressWarnings("deprecation")
+public class WorksetIterationsRecordApiCompilerTest extends CompilerTestBase {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String ITERATION_NAME = "Test Workset Iteration";
+ private static final String JOIN_WITH_INVARIANT_NAME = "Test Join Invariant";
+ private static final String JOIN_WITH_SOLUTION_SET = "Test Join SolutionSet";
+ private static final String NEXT_WORKSET_REDUCER_NAME = "Test Reduce Workset";
+ private static final String SOLUTION_DELTA_MAPPER_NAME = "Test Map Delta";
+
+ private final FieldList list0 = new FieldList(0);
+
+ @Test
+ public void testRecordApiWithDeferredSoltionSetUpdateWithMapper() {
+ Plan plan = getRecordTestPlan(false, true);
+
+ OptimizedPlan oPlan;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ return; // silence the compiler
+ }
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+ SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+
+ // verify reducer
+ assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
+ assertEquals(list0, worksetReducer.getKeys(0));
+
+ // currently, the system may partition before or after the mapper
+ ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
+ ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
+
+ assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
+ (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+
+ @Test
+ public void testRecordApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
+ Plan plan = getRecordTestPlan(false, false);
+
+ OptimizedPlan oPlan;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ return; // silence the compiler
+ }
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+
+ // verify reducer
+ assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
+ assertEquals(list0, worksetReducer.getKeys(0));
+
+
+ // verify solution delta
+ assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+
+ @Test
+ public void testRecordApiWithDirectSoltionSetUpdate() {
+ Plan plan = getRecordTestPlan(true, false);
+
+ OptimizedPlan oPlan;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ return; // silence the compiler
+ }
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
+ DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
+ SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
+
+ // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
+ // the in-loop partitioning is before the final reducer
+
+ // verify joinWithInvariant
+ assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
+ assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
+
+ // verify joinWithSolutionSet
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
+
+ // verify reducer
+ assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
+ assertEquals(list0, worksetReducer.getKeys(0));
+
+
+ // verify solution delta
+ assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
+ assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+
+ private Plan getRecordTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
+ FileDataSource solutionSetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Solution Set");
+ FileDataSource worksetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Workset");
+
+ FileDataSource invariantInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Invariant Input");
+
+ DeltaIteration iteration = new DeltaIteration(0, ITERATION_NAME);
+ iteration.setInitialSolutionSet(solutionSetInput);
+ iteration.setInitialWorkset(worksetInput);
+ iteration.setMaximumNumberOfIterations(100);
+
+ JoinOperator joinWithInvariant = JoinOperator.builder(new DummyMatchStub(), LongValue.class, 0, 0)
+ .input1(iteration.getWorkset())
+ .input2(invariantInput)
+ .name(JOIN_WITH_INVARIANT_NAME)
+ .build();
+
+ JoinOperator joinWithSolutionSet = JoinOperator.builder(
+ joinPreservesSolutionSet ? new DummyMatchStub() : new DummyNonPreservingMatchStub(), LongValue.class, 0, 0)
+ .input1(iteration.getSolutionSet())
+ .input2(joinWithInvariant)
+ .name(JOIN_WITH_SOLUTION_SET)
+ .build();
+
+ ReduceOperator nextWorkset = ReduceOperator.builder(new IdentityReduce(), LongValue.class, 0)
+ .input(joinWithSolutionSet)
+ .name(NEXT_WORKSET_REDUCER_NAME)
+ .build();
+
+ if (mapBeforeSolutionDelta) {
+ MapOperator mapper = MapOperator.builder(new IdentityMap())
+ .input(joinWithSolutionSet)
+ .name(SOLUTION_DELTA_MAPPER_NAME)
+ .build();
+ iteration.setSolutionSetDelta(mapper);
+ } else {
+ iteration.setSolutionSetDelta(joinWithSolutionSet);
+ }
+
+ iteration.setNextWorkset(nextWorkset);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, iteration, "Sink");
+
+ Plan plan = new Plan(sink);
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+ return plan;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
new file mode 100644
index 0000000..af03eec
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.costs;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.optimizer.dag.EstimateProvider;
+import org.junit.Test;
+
+/**
+ * Tests for the cost formulas in the {@link DefaultCostEstimator}. Most of the tests establish relative
+ * relationships.
+ */
+public class DefaultCostEstimatorTest {
+
+ // estimates
+
+ private static final long SMALL_DATA_SIZE = 10000;
+ private static final long SMALL_RECORD_COUNT = 100;
+
+ private static final long MEDIUM_DATA_SIZE = 500000000L;
+ private static final long MEDIUM_RECORD_COUNT = 500000L;
+
+ private static final long BIG_DATA_SIZE = 100000000000L;
+ private static final long BIG_RECORD_COUNT = 100000000L;
+
+ private static final EstimateProvider UNKNOWN_ESTIMATES = new UnknownEstimates();
+ private static final EstimateProvider ZERO_ESTIMATES = new Estimates(0, 0);
+ private static final EstimateProvider SMALL_ESTIMATES = new Estimates(SMALL_DATA_SIZE, SMALL_RECORD_COUNT);
+ private static final EstimateProvider MEDIUM_ESTIMATES = new Estimates(MEDIUM_DATA_SIZE, MEDIUM_RECORD_COUNT);
+ private static final EstimateProvider BIG_ESTIMATES = new Estimates(BIG_DATA_SIZE, BIG_RECORD_COUNT);
+
+ private final CostEstimator costEstimator = new DefaultCostEstimator();
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testShipStrategiesIsolated() {
+ testShipStrategiesIsolated(UNKNOWN_ESTIMATES, 1);
+ testShipStrategiesIsolated(UNKNOWN_ESTIMATES, 10);
+ testShipStrategiesIsolated(ZERO_ESTIMATES, 1);
+ testShipStrategiesIsolated(ZERO_ESTIMATES, 10);
+ testShipStrategiesIsolated(SMALL_ESTIMATES, 1);
+ testShipStrategiesIsolated(SMALL_ESTIMATES, 10);
+ testShipStrategiesIsolated(BIG_ESTIMATES, 1);
+ testShipStrategiesIsolated(BIG_ESTIMATES, 10);
+ }
+
+ private void testShipStrategiesIsolated(EstimateProvider estimates, int targetParallelism) {
+ Costs random = new Costs();
+ costEstimator.addRandomPartitioningCost(estimates, random);
+
+ Costs hash = new Costs();
+ costEstimator.addHashPartitioningCost(estimates, hash);
+
+ Costs range = new Costs();
+ costEstimator.addRangePartitionCost(estimates, range);
+
+ Costs broadcast = new Costs();
+ costEstimator.addBroadcastCost(estimates, targetParallelism, broadcast);
+
+ int randomVsHash = random.compareTo(hash);
+ int hashVsRange = hash.compareTo(range);
+ int hashVsBroadcast = hash.compareTo(broadcast);
+ int rangeVsBroadcast = range.compareTo(broadcast);
+
+ // repartition random is at most as expensive as hash partitioning
+ assertTrue(randomVsHash <= 0);
+
+ // range partitioning is always more expensive than hash partitioning
+ assertTrue(hashVsRange < 0);
+
+ // broadcasting is always more expensive than hash partitioning
+ if (targetParallelism > 1) {
+ assertTrue(hashVsBroadcast < 0);
+ } else {
+ assertTrue(hashVsBroadcast <= 0);
+ }
+
+ // range partitioning is not more expensive than broadcasting
+ if (targetParallelism > 1) {
+ assertTrue(rangeVsBroadcast < 0);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testShipStrategyCombinationsPlain() {
+ Costs hashBothSmall = new Costs();
+ Costs hashSmallAndLarge = new Costs();
+ Costs hashBothLarge = new Costs();
+
+ Costs hashSmallBcLarge10 = new Costs();
+ Costs hashLargeBcSmall10 = new Costs();
+
+ Costs hashSmallBcLarge1000 = new Costs();
+ Costs hashLargeBcSmall1000 = new Costs();
+
+ Costs forwardSmallBcLarge10 = new Costs();
+ Costs forwardLargeBcSmall10 = new Costs();
+
+ Costs forwardSmallBcLarge1000 = new Costs();
+ Costs forwardLargeBcSmall1000 = new Costs();
+
+ costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashBothSmall);
+ costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashBothSmall);
+
+ costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallAndLarge);
+ costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashSmallAndLarge);
+
+ costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashBothLarge);
+ costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashBothLarge);
+
+ costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallBcLarge10);
+ costEstimator.addBroadcastCost(BIG_ESTIMATES, 10, hashSmallBcLarge10);
+
+ costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashLargeBcSmall10);
+ costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 10, hashLargeBcSmall10);
+
+ costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallBcLarge1000);
+ costEstimator.addBroadcastCost(BIG_ESTIMATES, 1000, hashSmallBcLarge1000);
+
+ costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashLargeBcSmall1000);
+ costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 1000, hashLargeBcSmall1000);
+
+ costEstimator.addBroadcastCost(BIG_ESTIMATES, 10, forwardSmallBcLarge10);
+
+ costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 10, forwardLargeBcSmall10);
+
+ costEstimator.addBroadcastCost(BIG_ESTIMATES, 1000, forwardSmallBcLarge1000);
+
+ costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 1000, forwardLargeBcSmall1000);
+
+ // hash cost is roughly monotonous
+ assertTrue(hashBothSmall.compareTo(hashSmallAndLarge) < 0);
+ assertTrue(hashSmallAndLarge.compareTo(hashBothLarge) < 0);
+
+ // broadcast the smaller is better
+ assertTrue(hashLargeBcSmall10.compareTo(hashSmallBcLarge10) < 0);
+ assertTrue(forwardLargeBcSmall10.compareTo(forwardSmallBcLarge10) < 0);
+ assertTrue(hashLargeBcSmall1000.compareTo(hashSmallBcLarge1000) < 0);
+ assertTrue(forwardLargeBcSmall1000.compareTo(forwardSmallBcLarge1000) < 0);
+
+ // broadcasting small and forwarding large is better than partition both, given size difference
+ assertTrue(forwardLargeBcSmall10.compareTo(hashSmallAndLarge) < 0);
+
+ // broadcasting too far is expensive again
+ assertTrue(forwardLargeBcSmall1000.compareTo(hashSmallAndLarge) > 0);
+
+ // assert weight is respected
+ assertTrue(hashSmallBcLarge10.compareTo(hashSmallBcLarge1000) < 0);
+ assertTrue(hashLargeBcSmall10.compareTo(hashLargeBcSmall1000) < 0);
+ assertTrue(forwardSmallBcLarge10.compareTo(forwardSmallBcLarge1000) < 0);
+ assertTrue(forwardLargeBcSmall10.compareTo(forwardLargeBcSmall1000) < 0);
+
+ // forward versus hash
+ assertTrue(forwardSmallBcLarge10.compareTo(hashSmallBcLarge10) < 0);
+ assertTrue(forwardSmallBcLarge1000.compareTo(hashSmallBcLarge1000) < 0);
+ assertTrue(forwardLargeBcSmall10.compareTo(hashLargeBcSmall10) < 0);
+ assertTrue(forwardLargeBcSmall1000.compareTo(hashLargeBcSmall1000) < 0);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testShipStrategyCombinationsWithUnknowns() {
+ testShipStrategyCombinationsWithUnknowns(UNKNOWN_ESTIMATES);
+ testShipStrategyCombinationsWithUnknowns(ZERO_ESTIMATES);
+ testShipStrategyCombinationsWithUnknowns(SMALL_ESTIMATES);
+ testShipStrategyCombinationsWithUnknowns(MEDIUM_ESTIMATES);
+ testShipStrategyCombinationsWithUnknowns(BIG_ESTIMATES);
+ }
+
+ private void testShipStrategyCombinationsWithUnknowns(EstimateProvider knownEstimates) {
+ Costs hashBoth = new Costs();
+ Costs bcKnown10 = new Costs();
+ Costs bcUnknown10 = new Costs();
+ Costs bcKnown1000 = new Costs();
+ Costs bcUnknown1000 = new Costs();
+
+ costEstimator.addHashPartitioningCost(knownEstimates, hashBoth);
+ costEstimator.addHashPartitioningCost(UNKNOWN_ESTIMATES, hashBoth);
+
+ costEstimator.addBroadcastCost(knownEstimates, 10, bcKnown10);
+
+ costEstimator.addBroadcastCost(UNKNOWN_ESTIMATES, 10, bcUnknown10);
+
+ costEstimator.addBroadcastCost(knownEstimates, 1000, bcKnown1000);
+
+ costEstimator.addBroadcastCost(UNKNOWN_ESTIMATES, 1000, bcUnknown1000);
+
+ // if we do not know one of them, hashing both should be cheaper than anything
+ assertTrue(hashBoth.compareTo(bcKnown10) < 0);
+ assertTrue(hashBoth.compareTo(bcUnknown10) < 0);
+ assertTrue(hashBoth.compareTo(bcKnown1000) < 0);
+ assertTrue(hashBoth.compareTo(bcUnknown1000) < 0);
+
+ // there should be no bias in broadcasting a known or unknown size input
+ assertTrue(bcKnown10.compareTo(bcUnknown10) == 0);
+ assertTrue(bcKnown1000.compareTo(bcUnknown1000) == 0);
+
+ // replication factor does matter
+ assertTrue(bcKnown10.compareTo(bcKnown1000) < 0);
+ assertTrue(bcUnknown10.compareTo(bcUnknown1000) < 0);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testJoinCostFormulasPlain() {
+
+ // hash join costs
+
+ Costs hashBothSmall = new Costs();
+ Costs hashBothLarge = new Costs();
+ Costs hashSmallBuild = new Costs();
+ Costs hashLargeBuild = new Costs();
+
+ costEstimator.addHybridHashCosts(SMALL_ESTIMATES, BIG_ESTIMATES, hashSmallBuild, 1);
+ costEstimator.addHybridHashCosts(BIG_ESTIMATES, SMALL_ESTIMATES, hashLargeBuild, 1);
+ costEstimator.addHybridHashCosts(SMALL_ESTIMATES, SMALL_ESTIMATES, hashBothSmall, 1);
+ costEstimator.addHybridHashCosts(BIG_ESTIMATES, BIG_ESTIMATES, hashBothLarge, 1);
+
+ assertTrue(hashBothSmall.compareTo(hashSmallBuild) < 0);
+ assertTrue(hashSmallBuild.compareTo(hashLargeBuild) < 0);
+ assertTrue(hashLargeBuild.compareTo(hashBothLarge) < 0);
+
+ // merge join costs
+
+ Costs mergeBothSmall = new Costs();
+ Costs mergeBothLarge = new Costs();
+ Costs mergeSmallFirst = new Costs();
+ Costs mergeSmallSecond = new Costs();
+
+ costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallFirst);
+ costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallFirst);
+ costEstimator.addLocalMergeCost(SMALL_ESTIMATES, BIG_ESTIMATES, mergeSmallFirst, 1);
+
+ costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallSecond);
+ costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallSecond);
+ costEstimator.addLocalMergeCost(BIG_ESTIMATES, SMALL_ESTIMATES, mergeSmallSecond, 1);
+
+ costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
+ costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
+ costEstimator.addLocalMergeCost(SMALL_ESTIMATES, SMALL_ESTIMATES, mergeBothSmall, 1);
+
+ costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
+ costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
+ costEstimator.addLocalMergeCost(BIG_ESTIMATES, BIG_ESTIMATES, mergeBothLarge, 1);
+
+
+ assertTrue(mergeBothSmall.compareTo(mergeSmallFirst) < 0);
+ assertTrue(mergeBothSmall.compareTo(mergeSmallSecond) < 0);
+ assertTrue(mergeSmallFirst.compareTo(mergeSmallSecond) == 0);
+ assertTrue(mergeSmallFirst.compareTo(mergeBothLarge) < 0);
+ assertTrue(mergeSmallSecond.compareTo(mergeBothLarge) < 0);
+
+ // compare merge join and hash join costs
+
+ assertTrue(hashBothSmall.compareTo(mergeBothSmall) < 0);
+ assertTrue(hashBothLarge.compareTo(mergeBothLarge) < 0);
+ assertTrue(hashSmallBuild.compareTo(mergeSmallFirst) < 0);
+ assertTrue(hashSmallBuild.compareTo(mergeSmallSecond) < 0);
+ assertTrue(hashLargeBuild.compareTo(mergeSmallFirst) < 0);
+ assertTrue(hashLargeBuild.compareTo(mergeSmallSecond) < 0);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testJoinCostFormulasWithWeights() {
+ testJoinCostFormulasWithWeights(UNKNOWN_ESTIMATES, SMALL_ESTIMATES);
+ testJoinCostFormulasWithWeights(SMALL_ESTIMATES, UNKNOWN_ESTIMATES);
+ testJoinCostFormulasWithWeights(UNKNOWN_ESTIMATES, MEDIUM_ESTIMATES);
+ testJoinCostFormulasWithWeights(MEDIUM_ESTIMATES, UNKNOWN_ESTIMATES);
+ testJoinCostFormulasWithWeights(BIG_ESTIMATES, MEDIUM_ESTIMATES);
+ testJoinCostFormulasWithWeights(MEDIUM_ESTIMATES, BIG_ESTIMATES);
+ }
+
+ private void testJoinCostFormulasWithWeights(EstimateProvider e1, EstimateProvider e2) {
+ Costs hf1 = new Costs();
+ Costs hf5 = new Costs();
+ Costs hs1 = new Costs();
+ Costs hs5 = new Costs();
+ Costs mm1 = new Costs();
+ Costs mm5 = new Costs();
+
+ costEstimator.addHybridHashCosts(e1, e2, hf1, 1);
+ costEstimator.addHybridHashCosts(e1, e2, hf5, 5);
+ costEstimator.addHybridHashCosts(e2, e1, hs1, 1);
+ costEstimator.addHybridHashCosts(e2, e1, hs5, 5);
+
+ costEstimator.addLocalSortCost(e1, mm1);
+ costEstimator.addLocalSortCost(e2, mm1);
+ costEstimator.addLocalMergeCost(e1, e2, mm1, 1);
+
+ costEstimator.addLocalSortCost(e1, mm5);
+ costEstimator.addLocalSortCost(e2, mm5);
+ mm5.multiplyWith(5);
+ costEstimator.addLocalMergeCost(e1, e2, mm5, 5);
+
+ // weight 1 versus weight 5
+ assertTrue(hf1.compareTo(hf5) < 0);
+ assertTrue(hs1.compareTo(hs5) < 0);
+ assertTrue(mm1.compareTo(mm5) < 0);
+
+ // hash versus merge
+ assertTrue(hf1.compareTo(mm1) < 0);
+ assertTrue(hs1.compareTo(mm1) < 0);
+ assertTrue(hf5.compareTo(mm5) < 0);
+ assertTrue(hs5.compareTo(mm5) < 0);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ public void testHashJoinCostFormulasWithCaches() {
+
+ Costs hashBothUnknown10 = new Costs();
+ Costs hashBothUnknownCached10 = new Costs();
+
+ Costs hashBothSmall10 = new Costs();
+ Costs hashBothSmallCached10 = new Costs();
+
+ Costs hashSmallLarge10 = new Costs();
+ Costs hashSmallLargeCached10 = new Costs();
+
+ Costs hashLargeSmall10 = new Costs();
+ Costs hashLargeSmallCached10 = new Costs();
+
+ Costs hashLargeSmall1 = new Costs();
+ Costs hashLargeSmallCached1 = new Costs();
+
+ costEstimator.addHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknown10, 10);
+ costEstimator.addCachedHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknownCached10, 10);
+
+ costEstimator.addHybridHashCosts(MEDIUM_ESTIMATES, MEDIUM_ESTIMATES, hashBothSmall10, 10);
+ costEstimator.addCachedHybridHashCosts(MEDIUM_ESTIMATES, MEDIUM_ESTIMATES, hashBothSmallCached10, 10);
+
+ costEstimator.addHybridHashCosts(MEDIUM_ESTIMATES, BIG_ESTIMATES, hashSmallLarge10, 10);
+ costEstimator.addCachedHybridHashCosts(MEDIUM_ESTIMATES, BIG_ESTIMATES, hashSmallLargeCached10, 10);
+
+ costEstimator.addHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmall10, 10);
+ costEstimator.addCachedHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmallCached10, 10);
+
+ costEstimator.addHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmall1, 1);
+ costEstimator.addCachedHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmallCached1, 1);
+
+ // cached variant is always cheaper
+ assertTrue(hashBothUnknown10.compareTo(hashBothUnknownCached10) > 0);
+ assertTrue(hashBothSmall10.compareTo(hashBothSmallCached10) > 0);
+ assertTrue(hashSmallLarge10.compareTo(hashSmallLargeCached10) > 0);
+ assertTrue(hashLargeSmall10.compareTo(hashLargeSmallCached10) > 0);
+
+ // caching the large side is better, because then the small one is the one with additional I/O
+ assertTrue(hashLargeSmallCached10.compareTo(hashSmallLargeCached10) < 0);
+
+ // a weight of one makes the caching the same as the non-cached variant
+ assertTrue(hashLargeSmall1.compareTo(hashLargeSmallCached1) == 0);
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // Estimate providers
+ // --------------------------------------------------------------------------------------------
+
+ private static final class UnknownEstimates implements EstimateProvider {
+
+ @Override
+ public long getEstimatedOutputSize() { return -1; }
+
+ @Override
+ public long getEstimatedNumRecords() { return -1; }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() { return -1.0f; }
+ }
+
+ private static final class Estimates implements EstimateProvider {
+
+ private final long size;
+ private final long records;
+ private final float width;
+
+ public Estimates(long size, long records) {
+ this(size, records, -1.0f);
+ }
+
+ public Estimates(long size, long records, float width) {
+ this.size = size;
+ this.records = records;
+ this.width = width;
+ }
+
+ @Override
+ public long getEstimatedOutputSize() {
+ return this.size;
+ }
+
+ @Override
+ public long getEstimatedNumRecords() {
+ return this.records;
+ }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() {
+ return this.width;
+ }
+ }
+}
[51/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
index 85f1aa7..e4b35b7 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
@@ -38,7 +38,7 @@ import org.apache.flink.api.common.operators.SemanticProperties;
import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.costs.Costs;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
@@ -105,8 +105,8 @@ public class DataSourceNode extends OptimizerNode {
* @return The contract.
*/
@Override
- public GenericDataSourceBase<?, ?> getPactContract() {
- return (GenericDataSourceBase<?, ?>) super.getPactContract();
+ public GenericDataSourceBase<?, ?> getOperator() {
+ return (GenericDataSourceBase<?, ?>) super.getOperator();
}
@Override
@@ -123,8 +123,8 @@ public class DataSourceNode extends OptimizerNode {
}
@Override
- public List<PactConnection> getIncomingConnections() {
- return Collections.<PactConnection>emptyList();
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.<DagConnection>emptyList();
}
@Override
@@ -139,13 +139,13 @@ public class DataSourceNode extends OptimizerNode {
String inFormatDescription = "<unknown>";
try {
- format = getPactContract().getFormatWrapper().getUserCodeObject();
- Configuration config = getPactContract().getParameters();
+ format = getOperator().getFormatWrapper().getUserCodeObject();
+ Configuration config = getOperator().getParameters();
format.configure(config);
}
catch (Throwable t) {
- if (PactCompiler.LOG.isWarnEnabled()) {
- PactCompiler.LOG.warn("Could not instantiate InputFormat to obtain statistics."
+ if (Optimizer.LOG.isWarnEnabled()) {
+ Optimizer.LOG.warn("Could not instantiate InputFormat to obtain statistics."
+ " Limited statistics will be available.", t);
}
return;
@@ -158,7 +158,7 @@ public class DataSourceNode extends OptimizerNode {
}
// first of all, get the statistics from the cache
- final String statisticsKey = getPactContract().getStatisticsKey();
+ final String statisticsKey = getOperator().getStatisticsKey();
final BaseStatistics cachedStatistics = statistics.getBaseStatistics(statisticsKey);
BaseStatistics bs = null;
@@ -166,16 +166,16 @@ public class DataSourceNode extends OptimizerNode {
bs = format.getStatistics(cachedStatistics);
}
catch (Throwable t) {
- if (PactCompiler.LOG.isWarnEnabled()) {
- PactCompiler.LOG.warn("Error obtaining statistics from input format: " + t.getMessage(), t);
+ if (Optimizer.LOG.isWarnEnabled()) {
+ Optimizer.LOG.warn("Error obtaining statistics from input format: " + t.getMessage(), t);
}
}
if (bs != null) {
final long len = bs.getTotalInputSize();
if (len == BaseStatistics.SIZE_UNKNOWN) {
- if (PactCompiler.LOG.isInfoEnabled()) {
- PactCompiler.LOG.info("Compiler could not determine the size of input '" + inFormatDescription + "'. Using default estimates.");
+ if (Optimizer.LOG.isInfoEnabled()) {
+ Optimizer.LOG.info("Compiler could not determine the size of input '" + inFormatDescription + "'. Using default estimates.");
}
}
else if (len >= 0) {
@@ -207,14 +207,14 @@ public class DataSourceNode extends OptimizerNode {
return this.cachedPlans;
}
- SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getPactContract().getName()+")",
+ SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")",
this.gprops, this.lprops);
if(!replicatedInput) {
candidate.updatePropertiesWithUniqueSets(getUniqueFields());
final Costs costs = new Costs();
- if (FileInputFormat.class.isAssignableFrom(getPactContract().getFormatWrapper().getUserCodeClass()) &&
+ if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
this.estimatedOutputSize >= 0) {
estimator.addFileInputCost(this.estimatedOutputSize, costs);
}
@@ -223,10 +223,10 @@ public class DataSourceNode extends OptimizerNode {
// replicated input
final Costs costs = new Costs();
InputFormat<?,?> inputFormat =
- ((ReplicatingInputFormat<?,?>)getPactContract().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
+ ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
this.estimatedOutputSize >= 0) {
- estimator.addFileInputCost(this.estimatedOutputSize * this.getDegreeOfParallelism(), costs);
+ estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs);
}
candidate.setCosts(costs);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
index 5370678..482951b 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
@@ -16,10 +16,12 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.dag;
-
+/**
+ * Methods for operators / connections that provide estimated about data size and
+ * characteristics.
+ */
public interface EstimateProvider {
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
index c8dda12..118ddc8 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
@@ -41,8 +41,8 @@ public class FilterNode extends SingleInputNode {
}
@Override
- public FilterOperatorBase<?, ?> getPactContract() {
- return (FilterOperatorBase<?, ?>) super.getPactContract();
+ public FilterOperatorBase<?, ?> getOperator() {
+ return (FilterOperatorBase<?, ?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
index bb5799c..f713d56 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
@@ -40,8 +40,8 @@ public class FlatMapNode extends SingleInputNode {
}
@Override
- public FlatMapOperatorBase<?, ?, ?> getPactContract() {
- return (FlatMapOperatorBase<?, ?, ?>) super.getPactContract();
+ public FlatMapOperatorBase<?, ?, ?> getOperator() {
+ return (FlatMapOperatorBase<?, ?, ?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
index 2e2576d..564c0d3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
@@ -54,7 +54,7 @@ public class GroupCombineNode extends SingleInputNode {
private List<OperatorDescriptorSingle> initPossibleProperties() {
// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
- Ordering groupOrder = getPactContract().getGroupOrder();
+ Ordering groupOrder = getOperator().getGroupOrder();
if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
groupOrder = null;
}
@@ -74,8 +74,8 @@ public class GroupCombineNode extends SingleInputNode {
* @return The operator represented by this optimizer node.
*/
@Override
- public GroupCombineOperatorBase<?, ?, ?> getPactContract() {
- return (GroupCombineOperatorBase<?, ?, ?>) super.getPactContract();
+ public GroupCombineOperatorBase<?, ?, ?> getOperator() {
+ return (GroupCombineOperatorBase<?, ?, ?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
index 88ec53a..77acae5 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.operators.AllGroupReduceProperties;
import org.apache.flink.optimizer.operators.AllGroupWithPartialPreGroupProperties;
import org.apache.flink.optimizer.operators.GroupReduceProperties;
@@ -67,17 +67,17 @@ public class GroupReduceNode extends SingleInputNode {
private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
// see if an internal hint dictates the strategy to use
- final Configuration conf = getPactContract().getParameters();
- final String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
+ final Configuration conf = getOperator().getParameters();
+ final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
final boolean useCombiner;
if (localStrategy != null) {
- if (PactCompiler.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
useCombiner = false;
}
- else if (PactCompiler.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
+ else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
if (!isCombineable()) {
- PactCompiler.LOG.warn("Strategy hint for GroupReduce '" + getPactContract().getName() +
+ Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
"' requires combinable reduce, but user function is not marked combinable.");
}
useCombiner = true;
@@ -90,8 +90,8 @@ public class GroupReduceNode extends SingleInputNode {
// check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
Ordering groupOrder = null;
- if (getPactContract() instanceof GroupReduceOperatorBase) {
- groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getPactContract()).getGroupOrder();
+ if (getOperator() instanceof GroupReduceOperatorBase) {
+ groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getOperator()).getGroupOrder();
if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
groupOrder = null;
}
@@ -112,8 +112,8 @@ public class GroupReduceNode extends SingleInputNode {
* @return The operator represented by this optimizer node.
*/
@Override
- public GroupReduceOperatorBase<?, ?, ?> getPactContract() {
- return (GroupReduceOperatorBase<?, ?, ?>) super.getPactContract();
+ public GroupReduceOperatorBase<?, ?, ?> getOperator() {
+ return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
}
/**
@@ -123,7 +123,7 @@ public class GroupReduceNode extends SingleInputNode {
* @return True, if a combiner has been given, false otherwise.
*/
public boolean isCombineable() {
- return getPactContract().isCombinable();
+ return getOperator().isCombinable();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
index 3c67108..cbd58ca 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
@@ -27,7 +27,7 @@ import org.apache.flink.api.common.operators.base.JoinOperatorBase;
import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.operators.AbstractJoinDescriptor;
import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
@@ -62,8 +62,8 @@ public class JoinNode extends TwoInputNode {
* @return The contract.
*/
@Override
- public JoinOperatorBase<?, ?, ?, ?> getPactContract() {
- return (JoinOperatorBase<?, ?, ?, ?>) super.getPactContract();
+ public JoinOperatorBase<?, ?, ?, ?> getOperator() {
+ return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
}
@Override
@@ -116,21 +116,21 @@ public class JoinNode extends TwoInputNode {
{
// see if an internal hint dictates the strategy to use
Configuration conf = joinOperatorBase.getParameters();
- String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
if (localStrategy != null) {
final AbstractJoinDescriptor fixedDriverStrat;
- if (PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
{
fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
}
- else if (PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
+ else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
}
- else if (PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
+ else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
}
else {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
index cbcab2b..de3cd22 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.operators.base.JoinOperatorBase;
import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
@@ -58,8 +58,8 @@ public class MatchNode extends TwoInputNode {
* @return The contract.
*/
@Override
- public JoinOperatorBase<?, ?, ?, ?> getPactContract() {
- return (JoinOperatorBase<?, ?, ?, ?>) super.getPactContract();
+ public JoinOperatorBase<?, ?, ?, ?> getOperator() {
+ return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
}
@Override
@@ -110,19 +110,19 @@ public class MatchNode extends TwoInputNode {
private List<OperatorDescriptorDual> getDataProperties(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint) {
// see if an internal hint dictates the strategy to use
Configuration conf = joinOperatorBase.getParameters();
- String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
if (localStrategy != null) {
final OperatorDescriptorDual fixedDriverStrat;
- if (PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
- PactCompiler.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
{
fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
- } else if (PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
- } else if (PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
} else {
throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
index baeb1f7..0cad34e 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.dag;
import java.util.ArrayList;
@@ -48,8 +47,19 @@ import org.apache.flink.util.Visitable;
import org.apache.flink.util.Visitor;
/**
- * This class represents a node in the optimizer's internal representation of the PACT plan. It contains
- * extra information about estimates, hints and data properties.
+ * The OptimizerNode is the base class of all nodes in the optimizer DAG. The optimizer DAG is the
+ * optimizer's representation of a program, created before the actual optimization (which creates different
+ * candidate plans and computes their cost).
+ * <p>>
+ * Nodes in the DAG correspond (almost) one-to-one to the operators in a program. The optimizer DAG is constructed
+ * to hold the additional information that the optimizer needs:
+ * <ul>
+ * <li>Estimates of the data size processed by each operator</li>
+ * <li>Helper structures to track where the data flow "splits" and "joins", to support flows that are
+ * DAGs but not trees.</li>
+ * <li>Tags and weights to differentiate between loop-variant and -invariant parts of an iteration</li>
+ * <li>Interesting properties to be used during the enumeration of candidate plans</li>
+ * </ul>
*/
public abstract class OptimizerNode implements Visitable<OptimizerNode>, EstimateProvider, DumpableNode<OptimizerNode> {
@@ -59,13 +69,13 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
// Members
// --------------------------------------------------------------------------------------------
- private final Operator<?> pactContract; // The operator (Reduce / Join / DataSource / ...)
+ private final Operator<?> operator; // The operator (Reduce / Join / DataSource / ...)
private List<String> broadcastConnectionNames = new ArrayList<String>(); // the broadcast inputs names of this node
- private List<PactConnection> broadcastConnections = new ArrayList<PactConnection>(); // the broadcast inputs of this node
+ private List<DagConnection> broadcastConnections = new ArrayList<DagConnection>(); // the broadcast inputs of this node
- private List<PactConnection> outgoingConnections; // The links to succeeding nodes
+ private List<DagConnection> outgoingConnections; // The links to succeeding nodes
private InterestingProperties intProps; // the interesting properties of this node
@@ -88,7 +98,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
// --------------------------------- General Parameters ---------------------------------------
- private int degreeOfParallelism = -1; // the number of parallel instances of this node
+ private int parallelism = -1; // the number of parallel instances of this node
private long minimalMemoryPerSubTask = -1;
@@ -105,18 +115,17 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
// ------------------------------------------------------------------------
/**
- * Creates a new node for the optimizer plan.
+ * Creates a new optimizer node that represents the given program operator.
*
* @param op The operator that the node represents.
*/
public OptimizerNode(Operator<?> op) {
- this.pactContract = op;
+ this.operator = op;
readStubAnnotations();
}
protected OptimizerNode(OptimizerNode toCopy) {
- this.pactContract = toCopy.pactContract;
-
+ this.operator = toCopy.operator;
this.intProps = toCopy.intProps;
this.openBranches = toCopy.openBranches;
@@ -125,7 +134,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
this.estimatedOutputSize = toCopy.estimatedOutputSize;
this.estimatedNumRecords = toCopy.estimatedNumRecords;
- this.degreeOfParallelism = toCopy.degreeOfParallelism;
+ this.parallelism = toCopy.parallelism;
this.minimalMemoryPerSubTask = toCopy.minimalMemoryPerSubTask;
this.id = toCopy.id;
@@ -156,7 +165,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
ExecutionMode defaultExchangeMode);
/**
- * This function connects the predecessors to this operator.
+ * This function connects the operators that produce the broadcast inputs to this operator.
*
* @param operatorToNode The map from program operators to optimizer nodes.
* @param defaultExchangeMode The data exchange mode to use, if the operator does not
@@ -164,21 +173,19 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*
* @throws CompilerException
*/
- public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode)
- throws CompilerException
- {
+ public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) {
// skip for Operators that don't support broadcast variables
- if (!(getPactContract() instanceof AbstractUdfOperator<?, ?>)) {
+ if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) {
return;
}
// get all broadcast inputs
- AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getPactContract());
+ AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator());
// create connections and add them
for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) {
OptimizerNode predecessor = operatorToNode.get(input.getValue());
- PactConnection connection = new PactConnection(predecessor, this,
+ DagConnection connection = new DagConnection(predecessor, this,
ShipStrategyType.BROADCAST, defaultExchangeMode);
addBroadcastConnection(input.getKey(), connection);
predecessor.addOutgoingConnection(connection);
@@ -186,11 +193,12 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
}
/**
+ * Gets all incoming connections of this node.
* This method needs to be overridden by subclasses to return the children.
*
* @return The list of incoming connections.
*/
- public abstract List<PactConnection> getIncomingConnections();
+ public abstract List<DagConnection> getIncomingConnections();
/**
* Tells the node to compute the interesting properties for its inputs. The interesting properties
@@ -215,7 +223,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
List<UnclosedBranchDescriptor> branchesSoFar)
{
// handle the data flow branching for the broadcast inputs
- for (PactConnection broadcastInput : getBroadcastConnections()) {
+ for (DagConnection broadcastInput : getBroadcastConnections()) {
OptimizerNode bcSource = broadcastInput.getSource();
addClosedBranches(bcSource.closedBranchingNodes);
@@ -263,11 +271,11 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
public Iterable<OptimizerNode> getPredecessors() {
List<OptimizerNode> allPredecessors = new ArrayList<OptimizerNode>();
- for (PactConnection pactConnection : getIncomingConnections()) {
- allPredecessors.add(pactConnection.getSource());
+ for (DagConnection dagConnection : getIncomingConnections()) {
+ allPredecessors.add(dagConnection.getSource());
}
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
allPredecessors.add(conn.getSource());
}
@@ -306,7 +314,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*
* @param broadcastConnection The connection to add.
*/
- public void addBroadcastConnection(String name, PactConnection broadcastConnection) {
+ public void addBroadcastConnection(String name, DagConnection broadcastConnection) {
this.broadcastConnectionNames.add(name);
this.broadcastConnections.add(broadcastConnection);
}
@@ -321,26 +329,26 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
/**
* Return the list of inputs associated with broadcast variables for this node.
*/
- public List<PactConnection> getBroadcastConnections() {
+ public List<DagConnection> getBroadcastConnections() {
return this.broadcastConnections;
}
/**
* Adds a new outgoing connection to this node.
*
- * @param pactConnection
+ * @param connection
* The connection to add.
*/
- public void addOutgoingConnection(PactConnection pactConnection) {
+ public void addOutgoingConnection(DagConnection connection) {
if (this.outgoingConnections == null) {
- this.outgoingConnections = new ArrayList<PactConnection>();
+ this.outgoingConnections = new ArrayList<DagConnection>();
} else {
if (this.outgoingConnections.size() == 64) {
throw new CompilerException("Cannot currently handle nodes with more than 64 outputs.");
}
}
- this.outgoingConnections.add(pactConnection);
+ this.outgoingConnections.add(connection);
}
/**
@@ -348,7 +356,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*
* @return The list of outgoing connections.
*/
- public List<PactConnection> getOutgoingConnections() {
+ public List<DagConnection> getOutgoingConnections() {
return this.outgoingConnections;
}
@@ -357,8 +365,8 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*
* @return This node's operator.
*/
- public Operator<?> getPactContract() {
- return this.pactContract;
+ public Operator<?> getOperator() {
+ return this.operator;
}
/**
@@ -369,8 +377,8 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*
* @return The parallelism of the operator.
*/
- public int getDegreeOfParallelism() {
- return this.degreeOfParallelism;
+ public int getParallelism() {
+ return this.parallelism;
}
/**
@@ -386,7 +394,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
if (parallelism < 1 && parallelism != -1) {
throw new IllegalArgumentException("Degree of parallelism of " + parallelism + " is invalid.");
}
- this.degreeOfParallelism = parallelism;
+ this.parallelism = parallelism;
}
/**
@@ -395,7 +403,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
* @return The total amount of memory across all subtasks.
*/
public long getMinimalMemoryAcrossAllSubTasks() {
- return this.minimalMemoryPerSubTask == -1 ? -1 : this.minimalMemoryPerSubTask * this.degreeOfParallelism;
+ return this.minimalMemoryPerSubTask == -1 ? -1 : this.minimalMemoryPerSubTask * this.parallelism;
}
public boolean isOnDynamicPath() {
@@ -406,13 +414,13 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
boolean anyDynamic = false;
boolean allDynamic = true;
- for (PactConnection conn : getIncomingConnections()) {
+ for (DagConnection conn : getIncomingConnections()) {
boolean dynamicIn = conn.isOnDynamicPath();
anyDynamic |= dynamicIn;
allDynamic &= dynamicIn;
}
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
boolean dynamicIn = conn.isOnDynamicPath();
anyDynamic |= dynamicIn;
allDynamic &= dynamicIn;
@@ -424,7 +432,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
if (!allDynamic) {
// this node joins static and dynamic path.
// mark the connections where the source is not dynamic as cached
- for (PactConnection conn : getIncomingConnections()) {
+ for (DagConnection conn : getIncomingConnections()) {
if (!conn.getSource().isOnDynamicPath()) {
conn.setMaterializationMode(conn.getMaterializationMode().makeCached());
}
@@ -442,10 +450,10 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
public int getMaxDepth() {
int maxDepth = 0;
- for (PactConnection conn : getIncomingConnections()) {
+ for (DagConnection conn : getIncomingConnections()) {
maxDepth = Math.max(maxDepth, conn.getMaxDepth());
}
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
maxDepth = Math.max(maxDepth, conn.getMaxDepth());
}
@@ -502,7 +510,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
if (this.outgoingConnections == null) {
throw new IllegalStateException("The outgoing connections have not yet been initialized.");
}
- for (PactConnection conn : getOutgoingConnections()) {
+ for (DagConnection conn : getOutgoingConnections()) {
conn.markBreaksPipeline();
}
}
@@ -517,7 +525,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
* @return True, if on all outgoing connections, the interesting properties are set. False otherwise.
*/
public boolean haveAllOutputConnectionInterestingProperties() {
- for (PactConnection conn : getOutgoingConnections()) {
+ for (DagConnection conn : getOutgoingConnections()) {
if (conn.getInterestingProperties() == null) {
return false;
}
@@ -535,7 +543,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
* leaves the original objects, contained by the connections, unchanged.
*/
public void computeUnionOfInterestingPropertiesFromSuccessors() {
- List<PactConnection> conns = getOutgoingConnections();
+ List<DagConnection> conns = getOutgoingConnections();
if (conns.size() == 0) {
// no incoming, we have none ourselves
this.intProps = new InterestingProperties();
@@ -550,10 +558,10 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
public void clearInterestingProperties() {
this.intProps = null;
- for (PactConnection conn : getIncomingConnections()) {
+ for (DagConnection conn : getIncomingConnections()) {
conn.clearInterestingProperties();
}
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
conn.clearInterestingProperties();
}
}
@@ -570,7 +578,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
*/
public void computeOutputEstimates(DataStatistics statistics) {
// sanity checking
- for (PactConnection c : getIncomingConnections()) {
+ for (DagConnection c : getIncomingConnections()) {
if (c.getSource() == null) {
throw new CompilerException("Bug: Estimate computation called before inputs have been set.");
}
@@ -587,11 +595,11 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
}
// overwrite default estimates with hints, if given
- if (getPactContract() == null || getPactContract().getCompilerHints() == null) {
+ if (getOperator() == null || getOperator().getCompilerHints() == null) {
return ;
}
- CompilerHints hints = getPactContract().getCompilerHints();
+ CompilerHints hints = getOperator().getCompilerHints();
if (hints.getOutputSize() >= 0) {
this.estimatedOutputSize = hints.getOutputSize();
}
@@ -643,8 +651,8 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
}
protected void readUniqueFieldsAnnotation() {
- if (this.pactContract.getCompilerHints() != null) {
- Set<FieldSet> uniqueFieldSets = pactContract.getCompilerHints().getUniqueFields();
+ if (this.operator.getCompilerHints() != null) {
+ Set<FieldSet> uniqueFieldSets = operator.getCompilerHints().getUniqueFields();
if (uniqueFieldSets != null) {
if (this.uniqueFields == null) {
this.uniqueFields = new HashSet<FieldSet>();
@@ -686,7 +694,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
// for pruning, we are quasi AFTER the node, so in the presence of
// branches, we need form the per-branch-choice groups by the choice
- // they made at the latest unjoined branching node. Note that this is
+ // they made at the latest un-joined branching node. Note that this is
// different from the check for branch compatibility of candidates, as
// this happens on the input sub-plans and hence BEFORE the node (therefore
// it is relevant to find the latest (partially) joined branch point.
@@ -708,12 +716,12 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
@Override
public int compare(PlanNode o1, PlanNode o2) {
- for (int i = 0; i < branchDeterminers.length; i++) {
- PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminers[i]);
- PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminers[i]);
+ for (OptimizerNode branchDeterminer : branchDeterminers) {
+ PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminer);
+ PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminer);
int hash1 = System.identityHashCode(n1);
int hash2 = System.identityHashCode(n2);
-
+
if (hash1 != hash2) {
return hash1 - hash2;
}
@@ -775,8 +783,10 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
protected void prunePlanAlternativesWithCommonBranching(List<PlanNode> plans) {
// for each interesting property, which plans are cheapest
- final RequestedGlobalProperties[] gps = (RequestedGlobalProperties[]) this.intProps.getGlobalProperties().toArray(new RequestedGlobalProperties[this.intProps.getGlobalProperties().size()]);
- final RequestedLocalProperties[] lps = (RequestedLocalProperties[]) this.intProps.getLocalProperties().toArray(new RequestedLocalProperties[this.intProps.getLocalProperties().size()]);
+ final RequestedGlobalProperties[] gps = this.intProps.getGlobalProperties().toArray(
+ new RequestedGlobalProperties[this.intProps.getGlobalProperties().size()]);
+ final RequestedLocalProperties[] lps = this.intProps.getLocalProperties().toArray(
+ new RequestedLocalProperties[this.intProps.getLocalProperties().size()]);
final PlanNode[][] toKeep = new PlanNode[gps.length][];
final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
@@ -831,14 +841,12 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
plans.add(cheapest);
cheapest.setPruningMarker(); // remember that that plan is in the set
}
-
- // skip the top down delta cost check for now (TODO: implement this)
+
// add all others, which are optimal for some interesting properties
for (int i = 0; i < gps.length; i++) {
if (toKeep[i] != null) {
final PlanNode[] localMatches = toKeep[i];
- for (int k = 0; k < localMatches.length; k++) {
- final PlanNode n = localMatches[k];
+ for (final PlanNode n : localMatches) {
if (n != null && !n.isPruneMarkerSet()) {
n.setPruningMarker();
plans.add(n);
@@ -873,7 +881,7 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
}
- protected List<UnclosedBranchDescriptor> getBranchesForParent(PactConnection toParent) {
+ protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection toParent) {
if (this.outgoingConnections.size() == 1) {
// return our own stack of open branches, because nothing is added
if (this.openBranches == null || this.openBranches.isEmpty()) {
@@ -954,8 +962,8 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
* a) There is no branch in the sub-plan of this node
* b) Both candidates have the same candidate as the child at the last open branch.
*
- * @param plan1
- * @param plan2
+ * @param plan1 The root node of the first candidate plan.
+ * @param plan2 The root node of the second candidate plan.
* @return True if the nodes are branch compatible in the inputs.
*/
protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
@@ -1086,39 +1094,6 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
Collections.reverse(result);
return didCloseABranch;
}
-
- /**
- *
- */
- public static final class UnclosedBranchDescriptor
- {
- protected OptimizerNode branchingNode;
-
- protected long joinedPathsVector;
-
- /**
- * @param branchingNode
- * @param joinedPathsVector
- */
- protected UnclosedBranchDescriptor(OptimizerNode branchingNode, long joinedPathsVector)
- {
- this.branchingNode = branchingNode;
- this.joinedPathsVector = joinedPathsVector;
- }
-
- public OptimizerNode getBranchingNode() {
- return this.branchingNode;
- }
-
- public long getJoinedPathsVector() {
- return this.joinedPathsVector;
- }
-
- @Override
- public String toString() {
- return "(" + this.branchingNode.getPactContract() + ") [" + this.joinedPathsVector + "]";
- }
- }
@Override
public OptimizerNode getOptimizerNode() {
@@ -1145,14 +1120,53 @@ public abstract class OptimizerNode implements Visitable<OptimizerNode>, Estimat
StringBuilder bld = new StringBuilder();
bld.append(getName());
- bld.append(" (").append(getPactContract().getName()).append(") ");
+ bld.append(" (").append(getOperator().getName()).append(") ");
int i = 1;
- for (PactConnection conn : getIncomingConnections()) {
+ for (DagConnection conn : getIncomingConnections()) {
String shipStrategyName = conn.getShipStrategy() == null ? "null" : conn.getShipStrategy().name();
bld.append('(').append(i++).append(":").append(shipStrategyName).append(')');
}
return bld.toString();
}
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Description of an unclosed branch. An unclosed branch is when the data flow branched (one operator's
+ * result is consumed by multiple targets), but these different branches (targets) have not been joined
+ * together.
+ */
+ public static final class UnclosedBranchDescriptor {
+
+ protected OptimizerNode branchingNode;
+
+ protected long joinedPathsVector;
+
+ /**
+ * Creates a new branching descriptor.
+ *
+ * @param branchingNode The node where the branch occurred (teh node with multiple outputs).
+ * @param joinedPathsVector A bit vector describing which branches are tracked by this descriptor.
+ * The bit vector is one, where the branch is tracked, zero otherwise.
+ */
+ protected UnclosedBranchDescriptor(OptimizerNode branchingNode, long joinedPathsVector) {
+ this.branchingNode = branchingNode;
+ this.joinedPathsVector = joinedPathsVector;
+ }
+
+ public OptimizerNode getBranchingNode() {
+ return this.branchingNode;
+ }
+
+ public long getJoinedPathsVector() {
+ return this.joinedPathsVector;
+ }
+
+ @Override
+ public String toString() {
+ return "(" + this.branchingNode.getOperator() + ") [" + this.joinedPathsVector + "]";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PactConnection.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PactConnection.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PactConnection.java
deleted file mode 100644
index 661ceb5..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PactConnection.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.plandump.DumpableConnection;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-/**
- * A connection between to operators. Represents an intermediate result
- * and a data exchange between the two operators.
- *
- * The data exchange has a mode in which it performs (batch / pipelined)
- *
- * The data exchange strategy may be set on this connection, in which case
- * it is fixed and will not be determined during candidate plan enumeration.
- *
- * During the enumeration of interesting properties, this connection also holds
- * all interesting properties generated by the successor operator.
- */
-public class PactConnection implements EstimateProvider, DumpableConnection<OptimizerNode> {
-
- private final OptimizerNode source; // The source node of the connection
-
- private final OptimizerNode target; // The target node of the connection.
-
- private final ExecutionMode dataExchangeMode; // defines whether to use batch or pipelined data exchange
-
- private InterestingProperties interestingProps; // local properties that succeeding nodes are interested in
-
- private ShipStrategyType shipStrategy; // The data shipping strategy, if predefined.
-
- private TempMode materializationMode = TempMode.NONE; // the materialization mode
-
- private int maxDepth = -1;
-
- private boolean breakPipeline; // whether this connection should break the pipeline due to potential deadlocks
-
- /**
- * Creates a new Connection between two nodes. The shipping strategy is by default <tt>NONE</tt>.
- * The temp mode is by default <tt>NONE</tt>.
- *
- * @param source
- * The source node.
- * @param target
- * The target node.
- */
- public PactConnection(OptimizerNode source, OptimizerNode target, ExecutionMode exchangeMode) {
- this(source, target, null, exchangeMode);
- }
-
- /**
- * Creates a new Connection between two nodes.
- *
- * @param source
- * The source node.
- * @param target
- * The target node.
- * @param shipStrategy
- * The shipping strategy.
- * @param exchangeMode
- * The data exchange mode (pipelined / batch / batch only for shuffles / ... )
- */
- public PactConnection(OptimizerNode source, OptimizerNode target,
- ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
- {
- if (source == null || target == null) {
- throw new NullPointerException("Source and target must not be null.");
- }
- this.source = source;
- this.target = target;
- this.shipStrategy = shipStrategy;
- this.dataExchangeMode = exchangeMode;
- }
-
- /**
- * Constructor to create a result from an operator that is not
- * consumed by another operator.
- *
- * @param source
- * The source node.
- */
- public PactConnection(OptimizerNode source, ExecutionMode exchangeMode) {
- if (source == null) {
- throw new NullPointerException("Source and target must not be null.");
- }
- this.source = source;
- this.target = null;
- this.shipStrategy = ShipStrategyType.NONE;
- this.dataExchangeMode = exchangeMode;
- }
-
- /**
- * Gets the source of the connection.
- *
- * @return The source Node.
- */
- public OptimizerNode getSource() {
- return this.source;
- }
-
- /**
- * Gets the target of the connection.
- *
- * @return The target node.
- */
- public OptimizerNode getTarget() {
- return this.target;
- }
-
- /**
- * Gets the shipping strategy for this connection.
- *
- * @return The connection's shipping strategy.
- */
- public ShipStrategyType getShipStrategy() {
- return this.shipStrategy;
- }
-
- /**
- * Sets the shipping strategy for this connection.
- *
- * @param strategy
- * The shipping strategy to be applied to this connection.
- */
- public void setShipStrategy(ShipStrategyType strategy) {
- this.shipStrategy = strategy;
- }
-
- /**
- * Gets the data exchange mode to use for this connection.
- *
- * @return The data exchange mode to use for this connection.
- */
- public ExecutionMode getDataExchangeMode() {
- if (dataExchangeMode == null) {
- throw new IllegalStateException("This connection does not have the data exchange mode set");
- }
- return dataExchangeMode;
- }
-
- /**
- * Marks that this connection should do a decoupled data exchange (such as batched)
- * rather then pipeline data. Connections are marked as pipeline breakers to avoid
- * deadlock situations.
- */
- public void markBreaksPipeline() {
- this.breakPipeline = true;
- }
-
- /**
- * Checks whether this connection is marked to break the pipeline.
- *
- * @return True, if this connection is marked to break the pipeline, false otherwise.
- */
- public boolean isBreakingPipeline() {
- return this.breakPipeline;
- }
-
- /**
- * Gets the interesting properties object for this pact connection.
- * If the interesting properties for this connections have not yet been set,
- * this method returns null.
- *
- * @return The collection of all interesting properties, or null, if they have not yet been set.
- */
- public InterestingProperties getInterestingProperties() {
- return this.interestingProps;
- }
-
- /**
- * Sets the interesting properties for this pact connection.
- *
- * @param props The interesting properties.
- */
- public void setInterestingProperties(InterestingProperties props) {
- if (this.interestingProps == null) {
- this.interestingProps = props;
- } else {
- throw new IllegalStateException("Interesting Properties have already been set.");
- }
- }
-
- public void clearInterestingProperties() {
- this.interestingProps = null;
- }
-
- public void initMaxDepth() {
-
- if (this.maxDepth == -1) {
- this.maxDepth = this.source.getMaxDepth() + 1;
- } else {
- throw new IllegalStateException("Maximum path depth has already been initialized.");
- }
- }
-
- public int getMaxDepth() {
- if (this.maxDepth != -1) {
- return this.maxDepth;
- } else {
- throw new IllegalStateException("Maximum path depth has not been initialized.");
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Estimates
- // --------------------------------------------------------------------------------------------
-
- @Override
- public long getEstimatedOutputSize() {
- return this.source.getEstimatedOutputSize();
- }
-
- @Override
- public long getEstimatedNumRecords() {
- return this.source.getEstimatedNumRecords();
- }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() {
- return this.source.getEstimatedAvgWidthPerOutputRecord();
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- public TempMode getMaterializationMode() {
- return this.materializationMode;
- }
-
- public void setMaterializationMode(TempMode materializationMode) {
- this.materializationMode = materializationMode;
- }
-
- public boolean isOnDynamicPath() {
- return this.source.isOnDynamicPath();
- }
-
- public int getCostWeight() {
- return this.source.getCostWeight();
- }
-
- // --------------------------------------------------------------------------------------------
-
- public String toString() {
- StringBuilder buf = new StringBuilder(50);
- buf.append("Connection: ");
-
- if (this.source == null) {
- buf.append("null");
- } else {
- buf.append(this.source.getPactContract().getName());
- buf.append('(').append(this.source.getName()).append(')');
- }
-
- buf.append(" -> ");
-
- if (this.shipStrategy != null) {
- buf.append('[');
- buf.append(this.shipStrategy.name());
- buf.append(']').append(' ');
- }
-
- if (this.target == null) {
- buf.append("null");
- } else {
- buf.append(this.target.getPactContract().getName());
- buf.append('(').append(this.target.getName()).append(')');
- }
-
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
index 00b54ac..5c811b0 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
@@ -49,13 +49,13 @@ public class PartitionNode extends SingleInputNode {
super(operator);
OperatorDescriptorSingle descr = new PartitionDescriptor(
- this.getPactContract().getPartitionMethod(), this.keys, operator.getCustomPartitioner());
+ this.getOperator().getPartitionMethod(), this.keys, operator.getCustomPartitioner());
this.possibleProperties = Collections.singletonList(descr);
}
@Override
- public PartitionOperatorBase<?> getPactContract() {
- return (PartitionOperatorBase<?>) super.getPactContract();
+ public PartitionOperatorBase<?> getOperator() {
+ return (PartitionOperatorBase<?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
index 6f4c43d..1477038 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
@@ -62,8 +62,8 @@ public class ReduceNode extends SingleInputNode {
// ------------------------------------------------------------------------
@Override
- public ReduceOperatorBase<?, ?> getPactContract() {
- return (ReduceOperatorBase<?, ?>) super.getPactContract();
+ public ReduceOperatorBase<?, ?> getOperator() {
+ return (ReduceOperatorBase<?, ?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
index 9217beb..cc12bb8 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
@@ -36,7 +36,7 @@ import org.apache.flink.api.common.operators.SemanticProperties;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.util.FieldSet;
import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.InterestingProperties;
@@ -68,7 +68,7 @@ public abstract class SingleInputNode extends OptimizerNode {
protected final FieldSet keys; // The set of key fields
- protected PactConnection inConn; // the input of the node
+ protected DagConnection inConn; // the input of the node
// --------------------------------------------------------------------------------------------
@@ -103,8 +103,8 @@ public abstract class SingleInputNode extends OptimizerNode {
// --------------------------------------------------------------------------------------------
@Override
- public SingleInputOperator<?, ?, ?> getPactContract() {
- return (SingleInputOperator<?, ?, ?>) super.getPactContract();
+ public SingleInputOperator<?, ?, ?> getOperator() {
+ return (SingleInputOperator<?, ?, ?>) super.getOperator();
}
/**
@@ -112,7 +112,7 @@ public abstract class SingleInputNode extends OptimizerNode {
*
* @return The input.
*/
- public PactConnection getIncomingConnection() {
+ public DagConnection getIncomingConnection() {
return this.inConn;
}
@@ -121,7 +121,7 @@ public abstract class SingleInputNode extends OptimizerNode {
*
* @param inConn The input connection to set.
*/
- public void setIncomingConnection(PactConnection inConn) {
+ public void setIncomingConnection(DagConnection inConn) {
this.inConn = inConn;
}
@@ -139,14 +139,14 @@ public abstract class SingleInputNode extends OptimizerNode {
}
@Override
- public List<PactConnection> getIncomingConnections() {
+ public List<DagConnection> getIncomingConnections() {
return Collections.singletonList(this.inConn);
}
@Override
public SemanticProperties getSemanticProperties() {
- return getPactContract().getSemanticProperties();
+ return getOperator().getSemanticProperties();
}
@@ -155,18 +155,18 @@ public abstract class SingleInputNode extends OptimizerNode {
throws CompilerException
{
// see if an internal hint dictates the strategy to use
- final Configuration conf = getPactContract().getParameters();
- final String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null);
+ final Configuration conf = getOperator().getParameters();
+ final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
final ShipStrategyType preSet;
if (shipStrategy != null) {
- if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
+ if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
preSet = ShipStrategyType.PARTITION_HASH;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
preSet = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_FORWARD)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
preSet = ShipStrategyType.FORWARD;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
@@ -176,15 +176,15 @@ public abstract class SingleInputNode extends OptimizerNode {
}
// get the predecessor node
- Operator<?> children = ((SingleInputOperator<?, ?, ?>) getPactContract()).getInput();
+ Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
OptimizerNode pred;
- PactConnection conn;
+ DagConnection conn;
if (children == null) {
- throw new CompilerException("Error: Node for '" + getPactContract().getName() + "' has no input.");
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
} else {
pred = contractToNode.get(children);
- conn = new PactConnection(pred, this, defaultExchangeMode);
+ conn = new DagConnection(pred, this, defaultExchangeMode);
if (preSet != null) {
conn.setShipStrategy(preSet);
}
@@ -230,7 +230,7 @@ public abstract class SingleInputNode extends OptimizerNode {
}
this.inConn.setInterestingProperties(props);
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
conn.setInterestingProperties(new InterestingProperties());
}
}
@@ -251,11 +251,11 @@ public abstract class SingleInputNode extends OptimizerNode {
// calculate alternative sub-plans for broadcast inputs
final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
- List<PactConnection> broadcastConnections = getBroadcastConnections();
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
List<String> broadcastConnectionNames = getBroadcastConnectionNames();
for (int i = 0; i < broadcastConnections.size(); i++ ) {
- PactConnection broadcastConnection = broadcastConnections.get(i);
+ DagConnection broadcastConnection = broadcastConnections.get(i);
String broadcastConnectionName = broadcastConnectionNames.get(i);
List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
@@ -283,8 +283,8 @@ public abstract class SingleInputNode extends OptimizerNode {
final ExecutionMode executionMode = this.inConn.getDataExchangeMode();
- final int dop = getDegreeOfParallelism();
- final int inDop = getPredecessorNode().getDegreeOfParallelism();
+ final int dop = getParallelism();
+ final int inDop = getPredecessorNode().getParallelism();
final boolean dopChange = inDop != dop;
final boolean breaksPipeline = this.inConn.isBreakingPipeline();
@@ -509,7 +509,7 @@ public abstract class SingleInputNode extends OptimizerNode {
} else {
throw new CompilerException();
}
- for (PactConnection connection : getBroadcastConnections()) {
+ for (DagConnection connection : getBroadcastConnections()) {
connection.getSource().accept(visitor);
}
visitor.postVisit(this);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
index 2d65b4d..40725ba 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
@@ -31,8 +31,9 @@ import org.apache.flink.optimizer.util.NoOpBinaryUdfOp;
import org.apache.flink.types.Nothing;
/**
- * This class represents a utility node that is not part of the actual plan. It is used for plans with multiple data sinks to
- * transform it into a plan with a single root node. That way, the code that makes sure no costs are double-counted and that
+ * This class represents a utility node that is not part of the actual plan.
+ * It is used for plans with multiple data sinks to transform it into a plan with
+ * a single root node. That way, the code that makes sure no costs are double-counted and that
* candidate selection works correctly with nodes that have multiple outputs is transparently reused.
*/
public class SinkJoiner extends TwoInputNode {
@@ -40,8 +41,8 @@ public class SinkJoiner extends TwoInputNode {
public SinkJoiner(OptimizerNode input1, OptimizerNode input2) {
super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));
- PactConnection conn1 = new PactConnection(input1, this, null, ExecutionMode.PIPELINED);
- PactConnection conn2 = new PactConnection(input2, this, null, ExecutionMode.PIPELINED);
+ DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED);
+ DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED);
this.input1 = conn1;
this.input2 = conn2;
@@ -55,7 +56,7 @@ public class SinkJoiner extends TwoInputNode {
}
@Override
- public List<PactConnection> getOutgoingConnections() {
+ public List<DagConnection> getOutgoingConnections() {
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
index df47e56..1292cf5 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
@@ -46,7 +46,7 @@ public class SolutionSetNode extends AbstractPartialSolutionNode {
// --------------------------------------------------------------------------------------------
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
- this.cachedPlans = Collections.<PlanNode>singletonList(new SolutionSetPlanNode(this, "SolutionSet("+this.getPactContract().getName()+")", gProps, lProps, initialInput));
+ this.cachedPlans = Collections.<PlanNode>singletonList(new SolutionSetPlanNode(this, "SolutionSet("+this.getOperator().getName()+")", gProps, lProps, initialInput));
}
public SolutionSetPlanNode getCurrentSolutionSetPlanNode() {
@@ -74,8 +74,8 @@ public class SolutionSetNode extends AbstractPartialSolutionNode {
* @return The contract.
*/
@Override
- public SolutionSetPlaceHolder<?> getPactContract() {
- return (SolutionSetPlaceHolder<?>) super.getPactContract();
+ public SolutionSetPlaceHolder<?> getOperator() {
+ return (SolutionSetPlaceHolder<?>) super.getOperator();
}
@Override
@@ -89,7 +89,7 @@ public class SolutionSetNode extends AbstractPartialSolutionNode {
return;
}
- PactConnection solutionSetInput = this.iterationNode.getFirstIncomingConnection();
+ DagConnection solutionSetInput = this.iterationNode.getFirstIncomingConnection();
OptimizerNode solutionSetSource = solutionSetInput.getSource();
addClosedBranches(solutionSetSource.closedBranchingNodes);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
index 17bc8f1..83bc39a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
@@ -51,8 +51,8 @@ public class SortPartitionNode extends SingleInputNode {
}
@Override
- public SortPartitionOperatorBase<?> getPactContract() {
- return (SortPartitionOperatorBase<?>) super.getPactContract();
+ public SortPartitionOperatorBase<?> getOperator() {
+ return (SortPartitionOperatorBase<?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
index d51f2de..39da165 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
@@ -36,7 +36,7 @@ import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.SemanticProperties;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.InterestingProperties;
@@ -70,9 +70,9 @@ public abstract class TwoInputNode extends OptimizerNode {
protected final FieldList keys2; // The set of key fields for the second input
- protected PactConnection input1; // The first input edge
+ protected DagConnection input1; // The first input edge
- protected PactConnection input2; // The second input edge
+ protected DagConnection input2; // The second input edge
private List<OperatorDescriptorDual> cachedDescriptors;
@@ -109,8 +109,8 @@ public abstract class TwoInputNode extends OptimizerNode {
// ------------------------------------------------------------------------
@Override
- public DualInputOperator<?, ?, ?, ?> getPactContract() {
- return (DualInputOperator<?, ?, ?, ?>) super.getPactContract();
+ public DualInputOperator<?, ?, ?, ?> getOperator() {
+ return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
}
/**
@@ -118,7 +118,7 @@ public abstract class TwoInputNode extends OptimizerNode {
*
* @return The first input connection.
*/
- public PactConnection getFirstIncomingConnection() {
+ public DagConnection getFirstIncomingConnection() {
return this.input1;
}
@@ -127,7 +127,7 @@ public abstract class TwoInputNode extends OptimizerNode {
*
* @return The second input connection.
*/
- public PactConnection getSecondIncomingConnection() {
+ public DagConnection getSecondIncomingConnection() {
return this.input2;
}
@@ -148,8 +148,8 @@ public abstract class TwoInputNode extends OptimizerNode {
}
@Override
- public List<PactConnection> getIncomingConnections() {
- ArrayList<PactConnection> inputs = new ArrayList<PactConnection>(2);
+ public List<DagConnection> getIncomingConnections() {
+ ArrayList<DagConnection> inputs = new ArrayList<DagConnection>(2);
inputs.add(input1);
inputs.add(input2);
return inputs;
@@ -159,21 +159,21 @@ public abstract class TwoInputNode extends OptimizerNode {
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
- final Configuration conf = getPactContract().getParameters();
+ final Configuration conf = getOperator().getParameters();
ShipStrategyType preSet1 = null;
ShipStrategyType preSet2 = null;
- String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null);
+ String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
if (shipStrategy != null) {
- if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.FORWARD;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.BROADCAST;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
@@ -181,17 +181,17 @@ public abstract class TwoInputNode extends OptimizerNode {
}
// see if there is a hint that dictates which shipping strategy to use for the FIRST input
- shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
+ shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
if (shipStrategy != null) {
- if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet1 = ShipStrategyType.FORWARD;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet1 = ShipStrategyType.BROADCAST;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_HASH;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet1 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet1 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
@@ -199,17 +199,17 @@ public abstract class TwoInputNode extends OptimizerNode {
}
// see if there is a hint that dictates which shipping strategy to use for the SECOND input
- shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
+ shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
if (shipStrategy != null) {
- if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
preSet2 = ShipStrategyType.FORWARD;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
preSet2 = ShipStrategyType.BROADCAST;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_HASH;
- } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
preSet2 = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
preSet2 = ShipStrategyType.PARTITION_RANDOM;
} else {
throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
@@ -217,18 +217,18 @@ public abstract class TwoInputNode extends OptimizerNode {
}
// get the predecessors
- DualInputOperator<?, ?, ?, ?> contr = getPactContract();
+ DualInputOperator<?, ?, ?, ?> contr = getOperator();
Operator<?> leftPred = contr.getFirstInput();
Operator<?> rightPred = contr.getSecondInput();
OptimizerNode pred1;
- PactConnection conn1;
+ DagConnection conn1;
if (leftPred == null) {
- throw new CompilerException("Error: Node for '" + getPactContract().getName() + "' has no input set for first input.");
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
} else {
pred1 = contractToNode.get(leftPred);
- conn1 = new PactConnection(pred1, this, defaultExecutionMode);
+ conn1 = new DagConnection(pred1, this, defaultExecutionMode);
if (preSet1 != null) {
conn1.setShipStrategy(preSet1);
}
@@ -239,12 +239,12 @@ public abstract class TwoInputNode extends OptimizerNode {
pred1.addOutgoingConnection(conn1);
OptimizerNode pred2;
- PactConnection conn2;
+ DagConnection conn2;
if (rightPred == null) {
- throw new CompilerException("Error: Node for '" + getPactContract().getName() + "' has no input set for second input.");
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
} else {
pred2 = contractToNode.get(rightPred);
- conn2 = new PactConnection(pred2, this, defaultExecutionMode);
+ conn2 = new DagConnection(pred2, this, defaultExecutionMode);
if (preSet2 != null) {
conn2.setShipStrategy(preSet2);
}
@@ -290,7 +290,7 @@ public abstract class TwoInputNode extends OptimizerNode {
this.input1.setInterestingProperties(props1);
this.input2.setInterestingProperties(props2);
- for (PactConnection conn : getBroadcastConnections()) {
+ for (DagConnection conn : getBroadcastConnections()) {
conn.setInterestingProperties(new InterestingProperties());
}
}
@@ -314,11 +314,11 @@ public abstract class TwoInputNode extends OptimizerNode {
// calculate alternative sub-plans for broadcast inputs
final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
- List<PactConnection> broadcastConnections = getBroadcastConnections();
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
List<String> broadcastConnectionNames = getBroadcastConnectionNames();
for (int i = 0; i < broadcastConnections.size(); i++ ) {
- PactConnection broadcastConnection = broadcastConnections.get(i);
+ DagConnection broadcastConnection = broadcastConnections.get(i);
String broadcastConnectionName = broadcastConnectionNames.get(i);
List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
@@ -352,9 +352,9 @@ public abstract class TwoInputNode extends OptimizerNode {
final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
- final int dop = getDegreeOfParallelism();
- final int inDop1 = getFirstPredecessorNode().getDegreeOfParallelism();
- final int inDop2 = getSecondPredecessorNode().getDegreeOfParallelism();
+ final int dop = getParallelism();
+ final int inDop1 = getFirstPredecessorNode().getParallelism();
+ final int inDop2 = getSecondPredecessorNode().getParallelism();
final boolean dopChange1 = dop != inDop1;
final boolean dopChange2 = dop != inDop2;
@@ -720,7 +720,7 @@ public abstract class TwoInputNode extends OptimizerNode {
@Override
public SemanticProperties getSemanticProperties() {
- return getPactContract().getSemanticProperties();
+ return getOperator().getSemanticProperties();
}
// --------------------------------------------------------------------------------------------
@@ -737,7 +737,7 @@ public abstract class TwoInputNode extends OptimizerNode {
getFirstPredecessorNode().accept(visitor);
getSecondPredecessorNode().accept(visitor);
- for (PactConnection connection : getBroadcastConnections()) {
+ for (DagConnection connection : getBroadcastConnections()) {
connection.getSource().accept(visitor);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
index 2ec36b1..71a49f3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
@@ -32,7 +32,7 @@ import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.Optimizer.InterestingPropertyVisitor;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.InterestingProperties;
@@ -80,9 +80,9 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
private OptimizerNode nextWorkset;
- private PactConnection solutionSetDeltaRootConnection;
+ private DagConnection solutionSetDeltaRootConnection;
- private PactConnection nextWorksetRootConnection;
+ private DagConnection nextWorksetRootConnection;
private SingleRootJoiner singleRoot;
@@ -122,7 +122,7 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
// --------------------------------------------------------------------------------------------
public DeltaIterationBase<?, ?> getIterationContract() {
- return (DeltaIterationBase<?, ?>) getPactContract();
+ return (DeltaIterationBase<?, ?>) getOperator();
}
public SolutionSetNode getSolutionSetNode() {
@@ -167,9 +167,9 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
// if the next workset is equal to the workset, we need to inject a no-op node
if (nextWorkset == worksetNode || nextWorkset instanceof BinaryUnionNode) {
NoOpNode noop = new NoOpNode();
- noop.setDegreeOfParallelism(getDegreeOfParallelism());
+ noop.setDegreeOfParallelism(getParallelism());
- PactConnection noOpConn = new PactConnection(nextWorkset, noop, executionMode);
+ DagConnection noOpConn = new DagConnection(nextWorkset, noop, executionMode);
noop.setIncomingConnection(noOpConn);
nextWorkset.addOutgoingConnection(noOpConn);
@@ -179,9 +179,9 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
// attach an extra node to the solution set delta for the cases where we need to repartition
UnaryOperatorNode solutionSetDeltaUpdateAux = new UnaryOperatorNode("Solution-Set Delta", getSolutionSetKeyFields(),
new SolutionSetDeltaOperator(getSolutionSetKeyFields()));
- solutionSetDeltaUpdateAux.setDegreeOfParallelism(getDegreeOfParallelism());
+ solutionSetDeltaUpdateAux.setDegreeOfParallelism(getParallelism());
- PactConnection conn = new PactConnection(solutionSetDelta, solutionSetDeltaUpdateAux, executionMode);
+ DagConnection conn = new DagConnection(solutionSetDelta, solutionSetDeltaUpdateAux, executionMode);
solutionSetDeltaUpdateAux.setIncomingConnection(conn);
solutionSetDelta.addOutgoingConnection(conn);
@@ -189,10 +189,10 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
this.nextWorkset = nextWorkset;
this.singleRoot = new SingleRootJoiner();
- this.solutionSetDeltaRootConnection = new PactConnection(solutionSetDeltaUpdateAux,
+ this.solutionSetDeltaRootConnection = new DagConnection(solutionSetDeltaUpdateAux,
this.singleRoot, executionMode);
- this.nextWorksetRootConnection = new PactConnection(nextWorkset, this.singleRoot, executionMode);
+ this.nextWorksetRootConnection = new DagConnection(nextWorkset, this.singleRoot, executionMode);
this.singleRoot.setInputs(this.solutionSetDeltaRootConnection, this.nextWorksetRootConnection);
solutionSetDeltaUpdateAux.addOutgoingConnection(this.solutionSetDeltaRootConnection);
@@ -371,7 +371,7 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties",
FieldList.EMPTY_LIST);
- rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
+ rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(
rebuildWorksetPropertiesNode, "Rebuild Workset Properties",
@@ -454,7 +454,7 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
}
WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this,
- "WorksetIteration ("+this.getPactContract().getName()+")", solutionSetIn,
+ "WorksetIteration ("+this.getOperator().getName()+")", solutionSetIn,
worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
wsNode.initProperties(gp, lp);
@@ -566,7 +566,7 @@ public class WorksetIterationNode extends TwoInputNode implements IterationNode
setDegreeOfParallelism(1);
}
- public void setInputs(PactConnection input1, PactConnection input2) {
+ public void setInputs(DagConnection input1, DagConnection input2) {
this.input1 = input1;
this.input2 = input2;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
index bd39858..3b05aba 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
@@ -49,7 +49,7 @@ public class WorksetNode extends AbstractPartialSolutionNode {
if (this.cachedPlans != null) {
throw new IllegalStateException();
} else {
- WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getPactContract().getName()+")", gProps, lProps, initialInput);
+ WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
}
}
@@ -79,8 +79,8 @@ public class WorksetNode extends AbstractPartialSolutionNode {
* @return The contract.
*/
@Override
- public WorksetPlaceHolder<?> getPactContract() {
- return (WorksetPlaceHolder<?>) super.getPactContract();
+ public WorksetPlaceHolder<?> getOperator() {
+ return (WorksetPlaceHolder<?>) super.getOperator();
}
@Override
@@ -94,7 +94,7 @@ public class WorksetNode extends AbstractPartialSolutionNode {
return;
}
- PactConnection worksetInput = this.iterationNode.getSecondIncomingConnection();
+ DagConnection worksetInput = this.iterationNode.getSecondIncomingConnection();
OptimizerNode worksetSource = worksetInput.getSource();
addClosedBranches(worksetSource.closedBranchingNodes);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
index a547f04..57ba29d 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
@@ -185,11 +185,7 @@ public class GlobalProperties implements Cloneable {
}
public boolean isExactlyPartitionedOnFields(FieldList fields) {
- if (this.partitioning.isPartitionedOnKey() && fields.isExactMatch(this.partitioningFields)) {
- return true;
- } else {
- return false;
- }
+ return this.partitioning.isPartitionedOnKey() && fields.isExactMatch(this.partitioningFields);
}
public boolean matchesOrderedPartitioning(Ordering o) {
[24/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
deleted file mode 100644
index 0273659..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial","unchecked"})
-public class BinaryCustomPartitioningCompatibilityTest extends CompilerTestBase {
-
- @Test
- public void testCompatiblePartitioningJoin() {
- try {
- final Partitioner<Long> partitioner = new Partitioner<Long>() {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- };
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input1.partitionCustom(partitioner, 1)
- .join(input2.partitionCustom(partitioner, 0))
- .where(1).equalTo(0)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode partitioner1 = (SingleInputPlanNode) join.getInput1().getSource();
- SingleInputPlanNode partitioner2 = (SingleInputPlanNode) join.getInput2().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, join.getInput2().getShipStrategy());
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner1.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner2.getInput().getShipStrategy());
- assertEquals(partitioner, partitioner1.getInput().getPartitioner());
- assertEquals(partitioner, partitioner2.getInput().getPartitioner());
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCompatiblePartitioningCoGroup() {
- try {
- final Partitioner<Long> partitioner = new Partitioner<Long>() {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- };
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input1.partitionCustom(partitioner, 1)
- .coGroup(input2.partitionCustom(partitioner, 0))
- .where(1).equalTo(0)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode partitioner1 = (SingleInputPlanNode) coGroup.getInput1().getSource();
- SingleInputPlanNode partitioner2 = (SingleInputPlanNode) coGroup.getInput2().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, coGroup.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, coGroup.getInput2().getShipStrategy());
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner1.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner2.getInput().getShipStrategy());
- assertEquals(partitioner, partitioner1.getInput().getPartitioner());
- assertEquals(partitioner, partitioner2.getInput().getPartitioner());
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
deleted file mode 100644
index 08f7388..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CoGroupCustomPartitioningTest.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class CoGroupCustomPartitioningTest extends CompilerTestBase {
-
- @Test
- public void testCoGroupWithTuples() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input1
- .coGroup(input2)
- .where(1).equalTo(0)
- .withPartitioner(partitioner)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCoGroupWithTuplesWrongType() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- try {
- input1
- .coGroup(input2)
- .where(1).equalTo(0)
- .withPartitioner(partitioner);
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCoGroupWithPojos() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- input1
- .coGroup(input2)
- .where("b").equalTo("a")
- .withPartitioner(partitioner)
- .with(new DummyCoGroupFunction<Pojo2, Pojo3>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCoGroupWithPojosWrongType() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- try {
- input1
- .coGroup(input2)
- .where("a").equalTo("b")
- .withPartitioner(partitioner);
-
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCoGroupWithKeySelectors() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- input1
- .coGroup(input2)
- .where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector())
- .withPartitioner(partitioner)
- .with(new DummyCoGroupFunction<Pojo2, Pojo3>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCoGroupWithKeySelectorsWrongType() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- try {
- input1
- .coGroup(input2)
- .where(new Pojo2KeySelector()).equalTo(new Pojo3KeySelector())
- .withPartitioner(partitioner);
-
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testIncompatibleHashAndCustomPartitioning() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- DataSet<Tuple3<Long, Long, Long>> partitioned = input
- .partitionCustom(new Partitioner<Long>() {
- @Override
- public int partition(Long key, int numPartitions) { return 0; }
- }, 0)
- .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2");
-
-
- DataSet<Tuple3<Long, Long, Long>> grouped = partitioned
- .distinct(0, 1)
- .groupBy(1)
- .sortGroup(0, Order.ASCENDING)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1");
-
- grouped
- .coGroup(partitioned).where(0).equalTo(0)
- .with(new DummyCoGroupFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_HASH, coGroup.getInput1().getShipStrategy());
- assertTrue(coGroup.getInput2().getShipStrategy() == ShipStrategyType.PARTITION_HASH ||
- coGroup.getInput2().getShipStrategy() == ShipStrategyType.FORWARD);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-
- public static class Pojo2 {
- public int a;
- public int b;
- }
-
- public static class Pojo3 {
- public int a;
- public int b;
- public int c;
- }
-
- private static class Pojo2KeySelector implements KeySelector<Pojo2, Integer> {
- @Override
- public Integer getKey(Pojo2 value) {
- return value.a;
- }
- }
-
- private static class Pojo3KeySelector implements KeySelector<Pojo3, Integer> {
- @Override
- public Integer getKey(Pojo3 value) {
- return value.b;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
deleted file mode 100644
index 9fd676f..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningGlobalOptimizationTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class CustomPartitioningGlobalOptimizationTest extends CompilerTestBase {
-
- @Test
- public void testJoinReduceCombination() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- DataSet<Tuple3<Long, Long, Long>> joined = input1.join(input2)
- .where(1).equalTo(0)
- .projectFirst(0, 1)
- .<Tuple3<Long, Long, Long>>projectSecond(2)
- .withPartitioner(partitioner);
-
- joined.groupBy(1).withPartitioner(partitioner)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
-
- assertTrue("Reduce is not chained, property reuse does not happen",
- reducer.getInput().getSource() instanceof DualInputPlanNode);
-
- DualInputPlanNode join = (DualInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
-
- assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
deleted file mode 100644
index d397ea2..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityPartitionerMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class CustomPartitioningTest extends CompilerTestBase {
-
- @Test
- public void testPartitionTuples() {
- try {
- final Partitioner<Integer> part = new TestPartitionerInt();
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer,Integer>(0, 0))
- .rebalance();
-
- data
- .partitionCustom(part, 0)
- .mapPartition(new IdentityPartitionerMapper<Tuple2<Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode partitioner = (SingleInputPlanNode) mapper.getInput().getSource();
- SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getParallelism());
-
- assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
- assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getParallelism());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitionTuplesInvalidType() {
- try {
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer,Integer>(0, 0))
- .rebalance();
-
- try {
- data
- .partitionCustom(new TestPartitionerLong(), 0);
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitionPojo() {
- try {
- final Partitioner<Integer> part = new TestPartitionerInt();
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Pojo> data = env.fromElements(new Pojo())
- .rebalance();
-
- data
- .partitionCustom(part, "a")
- .mapPartition(new IdentityPartitionerMapper<Pojo>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode partitioner = (SingleInputPlanNode) mapper.getInput().getSource();
- SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getParallelism());
-
- assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
- assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getParallelism());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitionPojoInvalidType() {
- try {
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Pojo> data = env.fromElements(new Pojo())
- .rebalance();
-
- try {
- data
- .partitionCustom(new TestPartitionerLong(), "a");
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitionKeySelector() {
- try {
- final Partitioner<Integer> part = new TestPartitionerInt();
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Pojo> data = env.fromElements(new Pojo())
- .rebalance();
-
- data
- .partitionCustom(part, new TestKeySelectorInt<Pojo>())
- .mapPartition(new IdentityPartitionerMapper<Pojo>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode keyRemover = (SingleInputPlanNode) mapper.getInput().getSource();
- SingleInputPlanNode partitioner = (SingleInputPlanNode) keyRemover.getInput().getSource();
- SingleInputPlanNode keyExtractor = (SingleInputPlanNode) partitioner.getInput().getSource();
- SingleInputPlanNode balancer = (SingleInputPlanNode) keyExtractor.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getParallelism());
-
- assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getParallelism());
-
- assertEquals(ShipStrategyType.FORWARD, keyRemover.getInput().getShipStrategy());
- assertEquals(parallelism, keyRemover.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
- assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getParallelism());
-
- assertEquals(ShipStrategyType.FORWARD, keyExtractor.getInput().getShipStrategy());
- assertEquals(parallelism, keyExtractor.getParallelism());
-
- assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getParallelism());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitionKeySelectorInvalidType() {
- try {
- final Partitioner<Integer> part = (Partitioner<Integer>) (Partitioner<?>) new TestPartitionerLong();
- final int parallelism = 4;
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(parallelism);
-
- DataSet<Pojo> data = env.fromElements(new Pojo())
- .rebalance();
-
- try {
- data
- .partitionCustom(part, new TestKeySelectorInt<Pojo>());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static class Pojo {
- public int a;
- public int b;
- }
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestKeySelectorInt<T> implements KeySelector<T, Integer> {
- @Override
- public Integer getKey(T value) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
deleted file mode 100644
index 360487b..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingKeySelectorTranslationTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class GroupingKeySelectorTranslationTest extends CompilerTestBase {
-
- @Test
- public void testCustomPartitioningKeySelectorReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
- .withPartitioner(new TestPartitionerInt())
- .reduce(new DummyReducer<Tuple2<Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode keyRemovingMapper = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode reducer = (SingleInputPlanNode) keyRemovingMapper.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, keyRemovingMapper.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningKeySelectorGroupReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
- .withPartitioner(new TestPartitionerInt())
- .reduceGroup(new IdentityGroupReducer<Tuple2<Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningKeySelectorGroupReduceSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(new TestKeySelector<Tuple3<Integer,Integer,Integer>>())
- .withPartitioner(new TestPartitionerInt())
- .sortGroup(new TestKeySelector<Tuple3<Integer, Integer, Integer>>(), Order.ASCENDING)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Integer,Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningKeySelectorInvalidType() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data
- .groupBy(new TestKeySelector<Tuple2<Integer,Integer>>())
- .withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningKeySelectorInvalidTypeSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data
- .groupBy(new TestKeySelector<Tuple3<Integer,Integer,Integer>>())
- .sortGroup(1, Order.ASCENDING)
- .withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleRejectCompositeKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data
- .groupBy(new TestBinaryKeySelector<Tuple3<Integer,Integer,Integer>>())
- .withPartitioner(new TestPartitionerInt());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestKeySelector<T extends Tuple> implements KeySelector<T, Integer> {
- @Override
- public Integer getKey(T value) {
- return value.getField(0);
- }
- }
-
- private static class TestBinaryKeySelector<T extends Tuple> implements KeySelector<T, Tuple2<Integer, Integer>> {
- @Override
- public Tuple2<Integer, Integer> getKey(T value) {
- return new Tuple2<Integer, Integer>(value.<Integer>getField(0), value.<Integer>getField(1));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
deleted file mode 100644
index 8cd4809..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingPojoTranslationTest.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class GroupingPojoTranslationTest extends CompilerTestBase {
-
- @Test
- public void testCustomPartitioningTupleReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> data = env.fromElements(new Pojo2())
- .rebalance().setParallelism(4);
-
- data.groupBy("a").withPartitioner(new TestPartitionerInt())
- .reduce(new DummyReducer<Pojo2>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> data = env.fromElements(new Pojo2())
- .rebalance().setParallelism(4);
-
- data.groupBy("a").withPartitioner(new TestPartitionerInt())
- .reduceGroup(new IdentityGroupReducer<Pojo2>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduceSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo3> data = env.fromElements(new Pojo3())
- .rebalance().setParallelism(4);
-
- data.groupBy("a").withPartitioner(new TestPartitionerInt())
- .sortGroup("b", Order.ASCENDING)
- .reduceGroup(new IdentityGroupReducer<Pojo3>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduceSorted2() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo4> data = env.fromElements(new Pojo4())
- .rebalance().setParallelism(4);
-
- data.groupBy("a").withPartitioner(new TestPartitionerInt())
- .sortGroup("b", Order.ASCENDING)
- .sortGroup("c", Order.DESCENDING)
- .reduceGroup(new IdentityGroupReducer<Pojo4>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleInvalidType() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> data = env.fromElements(new Pojo2())
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy("a").withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleInvalidTypeSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo3> data = env.fromElements(new Pojo3())
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy("a")
- .sortGroup("b", Order.ASCENDING)
- .withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleRejectCompositeKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> data = env.fromElements(new Pojo2())
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy("a", "b")
- .withPartitioner(new TestPartitionerInt());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static class Pojo2 {
- public int a;
- public int b;
-
- }
-
- public static class Pojo3 {
- public int a;
- public int b;
- public int c;
- }
-
- public static class Pojo4 {
- public int a;
- public int b;
- public int c;
- public int d;
- }
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
deleted file mode 100644
index 779b8e5..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/GroupingTupleTranslationTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class GroupingTupleTranslationTest extends CompilerTestBase {
-
- @Test
- public void testCustomPartitioningTupleAgg() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(0).withPartitioner(new TestPartitionerInt())
- .sum(1)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(0).withPartitioner(new TestPartitionerInt())
- .reduce(new DummyReducer<Tuple2<Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(0).withPartitioner(new TestPartitionerInt())
- .reduceGroup(new IdentityGroupReducer<Tuple2<Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduceSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(0).withPartitioner(new TestPartitionerInt())
- .sortGroup(1, Order.ASCENDING)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Integer,Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleGroupReduceSorted2() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple4<Integer,Integer,Integer, Integer>> data = env.fromElements(new Tuple4<Integer,Integer,Integer,Integer>(0, 0, 0, 0))
- .rebalance().setParallelism(4);
-
- data.groupBy(0).withPartitioner(new TestPartitionerInt())
- .sortGroup(1, Order.ASCENDING)
- .sortGroup(2, Order.DESCENDING)
- .reduceGroup(new IdentityGroupReducer<Tuple4<Integer,Integer,Integer,Integer>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleInvalidType() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy(0).withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleInvalidTypeSorted() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy(0)
- .sortGroup(1, Order.ASCENDING)
- .withPartitioner(new TestPartitionerLong());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningTupleRejectCompositeKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0))
- .rebalance().setParallelism(4);
-
- try {
- data.groupBy(0, 1)
- .withPartitioner(new TestPartitionerInt());
- fail("Should throw an exception");
- }
- catch (InvalidProgramException e) {}
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
deleted file mode 100644
index eae40cf..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/JoinCustomPartitioningTest.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.custompartition;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class JoinCustomPartitioningTest extends CompilerTestBase {
-
- @Test
- public void testJoinWithTuples() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST).where(1).equalTo(0).withPartitioner(partitioner)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testJoinWithTuplesWrongType() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- try {
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST).where(1).equalTo(0)
- .withPartitioner(partitioner);
-
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testJoinWithPojos() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST)
- .where("b").equalTo("a").withPartitioner(partitioner)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testJoinWithPojosWrongType() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- try {
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST)
- .where("a").equalTo("b")
- .withPartitioner(partitioner);
-
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testJoinWithKeySelectors() {
- try {
- final Partitioner<Integer> partitioner = new TestPartitionerInt();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST)
- .where(new Pojo2KeySelector())
- .equalTo(new Pojo3KeySelector())
- .withPartitioner(partitioner)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_CUSTOM, join.getInput2().getShipStrategy());
- assertEquals(partitioner, join.getInput1().getPartitioner());
- assertEquals(partitioner, join.getInput2().getPartitioner());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testJoinWithKeySelectorsWrongType() {
- try {
- final Partitioner<Long> partitioner = new TestPartitionerLong();
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Pojo2> input1 = env.fromElements(new Pojo2());
- DataSet<Pojo3> input2 = env.fromElements(new Pojo3());
-
- try {
- input1
- .join(input2, JoinHint.REPARTITION_HASH_FIRST)
- .where(new Pojo2KeySelector())
- .equalTo(new Pojo3KeySelector())
- .withPartitioner(partitioner);
-
- fail("should throw an exception");
- }
- catch (InvalidProgramException e) {
- // expected
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testIncompatibleHashAndCustomPartitioning() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- DataSet<Tuple3<Long, Long, Long>> partitioned = input
- .partitionCustom(new Partitioner<Long>() {
- @Override
- public int partition(Long key, int numPartitions) { return 0; }
- }, 0)
- .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2");
-
-
- DataSet<Tuple3<Long, Long, Long>> grouped = partitioned
- .distinct(0, 1)
- .groupBy(1)
- .sortGroup(0, Order.ASCENDING)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1");
-
- grouped
- .join(partitioned, JoinHint.REPARTITION_HASH_FIRST).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple3<Long,Long,Long>>())
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode coGroup = (DualInputPlanNode) sink.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_HASH, coGroup.getInput1().getShipStrategy());
- assertTrue(coGroup.getInput2().getShipStrategy() == ShipStrategyType.PARTITION_HASH ||
- coGroup.getInput2().getShipStrategy() == ShipStrategyType.FORWARD);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static class TestPartitionerInt implements Partitioner<Integer> {
- @Override
- public int partition(Integer key, int numPartitions) {
- return 0;
- }
- }
-
- private static class TestPartitionerLong implements Partitioner<Long> {
- @Override
- public int partition(Long key, int numPartitions) {
- return 0;
- }
- }
-
- public static class Pojo2 {
- public int a;
- public int b;
- }
-
- public static class Pojo3 {
- public int a;
- public int b;
- public int c;
- }
-
- private static class Pojo2KeySelector implements KeySelector<Pojo2, Integer> {
- @Override
- public Integer getKey(Pojo2 value) {
- return value.a;
- }
- }
-
- private static class Pojo3KeySelector implements KeySelector<Pojo3, Integer> {
- @Override
- public Integer getKey(Pojo3 value) {
- return value.b;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
deleted file mode 100644
index cb4bd78..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataexchange;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityFlatMapper;
-import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
-import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.junit.Test;
-
-import java.util.Collection;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This test checks the correct assignment of the DataExchangeMode to
- * connections for programs that branch, and re-join those branches.
- *
- * <pre>
- * /-> (sink)
- * /
- * /-> (reduce) -+ /-> (flatmap) -> (sink)
- * / \ /
- * (source) -> (map) - (join) -+-----\
- * \ / \
- * \-> (filter) -+ \
- * \ (co group) -> (sink)
- * \ /
- * \-> (reduce) - /
- * </pre>
- */
-@SuppressWarnings("serial")
-public class DataExchangeModeClosedBranchingTest extends CompilerTestBase {
-
- @Test
- public void testPipelinedForced() {
- // PIPELINED_FORCED should result in pipelining all the way
- verifyBranchingJoiningPlan(ExecutionMode.PIPELINED_FORCED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testPipelined() {
- // PIPELINED should result in pipelining all the way
- verifyBranchingJoiningPlan(ExecutionMode.PIPELINED,
- DataExchangeMode.PIPELINED, // to map
- DataExchangeMode.PIPELINED, // to combiner connections are pipelined
- DataExchangeMode.BATCH, // to reduce
- DataExchangeMode.BATCH, // to filter
- DataExchangeMode.PIPELINED, // to sink after reduce
- DataExchangeMode.PIPELINED, // to join (first input)
- DataExchangeMode.BATCH, // to join (second input)
- DataExchangeMode.PIPELINED, // combiner connections are pipelined
- DataExchangeMode.BATCH, // to other reducer
- DataExchangeMode.PIPELINED, // to flatMap
- DataExchangeMode.PIPELINED, // to sink after flatMap
- DataExchangeMode.PIPELINED, // to coGroup (first input)
- DataExchangeMode.PIPELINED, // to coGroup (second input)
- DataExchangeMode.PIPELINED // to sink after coGroup
- );
- }
-
- @Test
- public void testBatch() {
- // BATCH should result in batching the shuffle all the way
- verifyBranchingJoiningPlan(ExecutionMode.BATCH,
- DataExchangeMode.PIPELINED, // to map
- DataExchangeMode.PIPELINED, // to combiner connections are pipelined
- DataExchangeMode.BATCH, // to reduce
- DataExchangeMode.BATCH, // to filter
- DataExchangeMode.PIPELINED, // to sink after reduce
- DataExchangeMode.BATCH, // to join (first input)
- DataExchangeMode.BATCH, // to join (second input)
- DataExchangeMode.PIPELINED, // combiner connections are pipelined
- DataExchangeMode.BATCH, // to other reducer
- DataExchangeMode.PIPELINED, // to flatMap
- DataExchangeMode.PIPELINED, // to sink after flatMap
- DataExchangeMode.BATCH, // to coGroup (first input)
- DataExchangeMode.BATCH, // to coGroup (second input)
- DataExchangeMode.PIPELINED // to sink after coGroup
- );
- }
-
- @Test
- public void testBatchForced() {
- // BATCH_FORCED should result in batching all the way
- verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
- DataExchangeMode.BATCH, // to map
- DataExchangeMode.PIPELINED, // to combiner connections are pipelined
- DataExchangeMode.BATCH, // to reduce
- DataExchangeMode.BATCH, // to filter
- DataExchangeMode.BATCH, // to sink after reduce
- DataExchangeMode.BATCH, // to join (first input)
- DataExchangeMode.BATCH, // to join (second input)
- DataExchangeMode.PIPELINED, // combiner connections are pipelined
- DataExchangeMode.BATCH, // to other reducer
- DataExchangeMode.BATCH, // to flatMap
- DataExchangeMode.BATCH, // to sink after flatMap
- DataExchangeMode.BATCH, // to coGroup (first input)
- DataExchangeMode.BATCH, // to coGroup (second input)
- DataExchangeMode.BATCH // to sink after coGroup
- );
- }
-
- private void verifyBranchingJoiningPlan(ExecutionMode execMode,
- DataExchangeMode toMap,
- DataExchangeMode toReduceCombiner,
- DataExchangeMode toReduce,
- DataExchangeMode toFilter,
- DataExchangeMode toReduceSink,
- DataExchangeMode toJoin1,
- DataExchangeMode toJoin2,
- DataExchangeMode toOtherReduceCombiner,
- DataExchangeMode toOtherReduce,
- DataExchangeMode toFlatMap,
- DataExchangeMode toFlatMapSink,
- DataExchangeMode toCoGroup1,
- DataExchangeMode toCoGroup2,
- DataExchangeMode toCoGroupSink)
- {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().setExecutionMode(execMode);
-
- DataSet<Tuple2<Long, Long>> data = env.fromElements(33L, 44L)
- .map(new MapFunction<Long, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Long value) {
- return new Tuple2<Long, Long>(value, value);
- }
- });
-
- DataSet<Tuple2<Long, Long>> reduced = data.groupBy(0).reduce(new SelectOneReducer<Tuple2<Long, Long>>());
- reduced.output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("reduceSink");
-
- DataSet<Tuple2<Long, Long>> filtered = data.filter(new FilterFunction<Tuple2<Long, Long>>() {
- @Override
- public boolean filter(Tuple2<Long, Long> value) throws Exception {
- return false;
- }
- });
-
- DataSet<Tuple2<Long, Long>> joined = reduced.join(filtered)
- .where(1).equalTo(1)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- joined.flatMap(new IdentityFlatMapper<Tuple2<Long, Long>>())
- .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("flatMapSink");
-
- joined.coGroup(filtered.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()))
- .where(0).equalTo(0)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
- .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()).name("cgSink");
-
-
- OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
-
- SinkPlanNode reduceSink = findSink(optPlan.getDataSinks(), "reduceSink");
- SinkPlanNode flatMapSink = findSink(optPlan.getDataSinks(), "flatMapSink");
- SinkPlanNode cgSink = findSink(optPlan.getDataSinks(), "cgSink");
-
- DualInputPlanNode coGroupNode = (DualInputPlanNode) cgSink.getPredecessor();
-
- DualInputPlanNode joinNode = (DualInputPlanNode) coGroupNode.getInput1().getSource();
- SingleInputPlanNode otherReduceNode = (SingleInputPlanNode) coGroupNode.getInput2().getSource();
- SingleInputPlanNode otherReduceCombinerNode = (SingleInputPlanNode) otherReduceNode.getPredecessor();
-
- SingleInputPlanNode reduceNode = (SingleInputPlanNode) joinNode.getInput1().getSource();
- SingleInputPlanNode reduceCombinerNode = (SingleInputPlanNode) reduceNode.getPredecessor();
- assertEquals(reduceNode, reduceSink.getPredecessor());
-
- SingleInputPlanNode filterNode = (SingleInputPlanNode) joinNode.getInput2().getSource();
- assertEquals(filterNode, otherReduceCombinerNode.getPredecessor());
-
- SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
- assertEquals(mapNode, reduceCombinerNode.getPredecessor());
-
- SingleInputPlanNode flatMapNode = (SingleInputPlanNode) flatMapSink.getPredecessor();
- assertEquals(joinNode, flatMapNode.getPredecessor());
-
- // verify the data exchange modes
-
- assertEquals(toReduceSink, reduceSink.getInput().getDataExchangeMode());
- assertEquals(toFlatMapSink, flatMapSink.getInput().getDataExchangeMode());
- assertEquals(toCoGroupSink, cgSink.getInput().getDataExchangeMode());
-
- assertEquals(toCoGroup1, coGroupNode.getInput1().getDataExchangeMode());
- assertEquals(toCoGroup2, coGroupNode.getInput2().getDataExchangeMode());
-
- assertEquals(toJoin1, joinNode.getInput1().getDataExchangeMode());
- assertEquals(toJoin2, joinNode.getInput2().getDataExchangeMode());
-
- assertEquals(toOtherReduce, otherReduceNode.getInput().getDataExchangeMode());
- assertEquals(toOtherReduceCombiner, otherReduceCombinerNode.getInput().getDataExchangeMode());
-
- assertEquals(toFlatMap, flatMapNode.getInput().getDataExchangeMode());
-
- assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
- assertEquals(toReduce, reduceNode.getInput().getDataExchangeMode());
- assertEquals(toReduceCombiner, reduceCombinerNode.getInput().getDataExchangeMode());
-
- assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
- for (SinkPlanNode node : collection) {
- String nodeName = node.getOptimizerNode().getOperator().getName();
- if (nodeName != null && nodeName.equals(name)) {
- return node;
- }
- }
-
- throw new IllegalArgumentException("No node with that name was found.");
- }
-}
[20/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/DataStatistics.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/DataStatistics.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/DataStatistics.java
new file mode 100644
index 0000000..cf6f4ec
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/DataStatistics.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+
+/**
+ * The collection of access methods that can be used to retrieve statistical information about the
+ * data processed in a job. Currently this method acts as an entry point only for obtaining cached
+ * statistics.
+ */
+public class DataStatistics {
+
+ private final Map<String, BaseStatistics> baseStatisticsCache;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new statistics object, with an empty cache.
+ */
+ public DataStatistics() {
+ this.baseStatisticsCache = new HashMap<String, BaseStatistics>();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the base statistics for the input identified by the given identifier.
+ *
+ * @param inputIdentifier The identifier for the input.
+ * @return The statistics that were cached for this input.
+ */
+ public BaseStatistics getBaseStatistics(String inputIdentifier) {
+ synchronized (this.baseStatisticsCache) {
+ return this.baseStatisticsCache.get(inputIdentifier);
+ }
+ }
+
+ /**
+ * Caches the given statistics. They are later retrievable under the given identifier.
+ *
+ * @param statistics The statistics to cache.
+ * @param identifier The identifier which may be later used to retrieve the statistics.
+ */
+ public void cacheBaseStatistics(BaseStatistics statistics, String identifier) {
+ synchronized (this.baseStatisticsCache) {
+ this.baseStatisticsCache.put(identifier, statistics);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/Optimizer.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/Optimizer.java
new file mode 100644
index 0000000..2101428
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/Optimizer.java
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.flink.optimizer.traversals.BinaryUnionReplacer;
+import org.apache.flink.optimizer.traversals.BranchesVisitor;
+import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
+import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.traversals.PlanFinalizer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.costs.DefaultCostEstimator;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SinkJoiner;
+import org.apache.flink.optimizer.deadlockdetect.DeadlockPreventer;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.postpass.OptimizerPostPass;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.util.InstantiationUtil;
+
+/**
+ * The optimizer that takes the user specified program plan and creates an optimized plan that contains
+ * exact descriptions about how the physical execution will take place. It first translates the user
+ * program into an internal optimizer representation and then chooses between different alternatives
+ * for shipping strategies and local strategies.
+ * <p>
+ * The basic principle is taken from optimizer works in systems such as Volcano/Cascades and Selinger/System-R/DB2. The
+ * optimizer walks from the sinks down, generating interesting properties, and ascends from the sources generating
+ * alternative plans, pruning against the interesting properties.
+ * <p>
+ * The optimizer also assigns the memory to the individual tasks. This is currently done in a very simple fashion: All
+ * sub-tasks that need memory (e.g. reduce or join) are given an equal share of memory.
+ */
+public class Optimizer {
+
+ // ------------------------------------------------------------------------
+ // Constants
+ // ------------------------------------------------------------------------
+
+ /**
+ * Compiler hint key for the input channel's shipping strategy. This String is a key to the operator's stub
+ * parameters. The corresponding value tells the compiler which shipping strategy to use for the input channel.
+ * If the operator has two input channels, the shipping strategy is applied to both input channels.
+ */
+ public static final String HINT_SHIP_STRATEGY = "INPUT_SHIP_STRATEGY";
+
+ /**
+ * Compiler hint key for the <b>first</b> input channel's shipping strategy. This String is a key to
+ * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
+ * to use for the <b>first</b> input channel. Only applicable to operators with two inputs.
+ */
+ public static final String HINT_SHIP_STRATEGY_FIRST_INPUT = "INPUT_LEFT_SHIP_STRATEGY";
+
+ /**
+ * Compiler hint key for the <b>second</b> input channel's shipping strategy. This String is a key to
+ * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
+ * to use for the <b>second</b> input channel. Only applicable to operators with two inputs.
+ */
+ public static final String HINT_SHIP_STRATEGY_SECOND_INPUT = "INPUT_RIGHT_SHIP_STRATEGY";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a <b>Forward</b> strategy on the
+ * input channel, i.e. no redistribution of any kind.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_FORWARD = "SHIP_FORWARD";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a random repartition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION= "SHIP_REPARTITION";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a hash-partition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION_HASH = "SHIP_REPARTITION_HASH";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a range-partition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION_RANGE = "SHIP_REPARTITION_RANGE";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a <b>broadcast</b> strategy on the
+ * input channel.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_BROADCAST = "SHIP_BROADCAST";
+
+ /**
+ * Compiler hint key for the operator's local strategy. This String is a key to the operator's stub
+ * parameters. The corresponding value tells the compiler which local strategy to use to process the
+ * data inside one partition.
+ * <p>
+ * This hint is ignored by operators that do not have a local strategy (such as <i>Map</i>), or by operators that
+ * have no choice in their local strategy (such as <i>Cross</i>).
+ */
+ public static final String HINT_LOCAL_STRATEGY = "LOCAL_STRATEGY";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
+ * For example, a <i>Reduce</i> operator will sort the data to group it.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT = "LOCAL_STRATEGY_SORT";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
+ * During sorting a combine method is repeatedly applied to reduce the data volume.
+ * For example, a <i>Reduce</i> operator will sort the data to group it.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_COMBINING_SORT = "LOCAL_STRATEGY_COMBINING_SORT";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy on both
+ * inputs with subsequent merging of inputs.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE = "LOCAL_STRATEGY_SORT_BOTH_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
+ * The first input is sorted, the second input is assumed to be sorted. After sorting both inputs are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE = "LOCAL_STRATEGY_SORT_FIRST_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
+ * The second input is sorted, the first input is assumed to be sorted. After sorting both inputs are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE = "LOCAL_STRATEGY_SORT_SECOND_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>merge based</b> local strategy.
+ * Both inputs are assumed to be sorted and are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_MERGE = "LOCAL_STRATEGY_MERGE";
+
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
+ * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
+ * matching keys. The <b>first</b> input will be used to build the hash table, the second input will be
+ * used to probe the table.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST = "LOCAL_STRATEGY_HASH_BUILD_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
+ * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
+ * matching keys. The <b>second</b> input will be used to build the hash table, the first input will be
+ * used to probe the table.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND = "LOCAL_STRATEGY_HASH_BUILD_SECOND";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
+ * Hence, the data of the first input will be is streamed though, while the data of the second input is stored on
+ * disk
+ * and repeatedly read.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
+ * Hence, the data of the second input will be is streamed though, while the data of the first input is stored on
+ * disk
+ * and repeatedly read.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
+ * Further more, the first input, being the outer side, will be processed in blocks, and for each block, the second
+ * input,
+ * being the inner side, will read repeatedly from disk.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
+ * Further more, the second input, being the outer side, will be processed in blocks, and for each block, the first
+ * input,
+ * being the inner side, will read repeatedly from disk.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND";
+
+ /**
+ * The log handle that is used by the compiler to log messages.
+ */
+ public static final Logger LOG = LoggerFactory.getLogger(Optimizer.class);
+
+ // ------------------------------------------------------------------------
+ // Members
+ // ------------------------------------------------------------------------
+
+ /**
+ * The statistics object used to obtain statistics, such as input sizes,
+ * for the cost estimation process.
+ */
+ private final DataStatistics statistics;
+
+ /**
+ * The cost estimator used by the compiler.
+ */
+ private final CostEstimator costEstimator;
+
+ /**
+ * The default degree of parallelism for jobs compiled by this compiler.
+ */
+ private int defaultDegreeOfParallelism;
+
+
+ // ------------------------------------------------------------------------
+ // Constructor & Setup
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new optimizer instance. The optimizer has no access to statistics about the
+ * inputs and can hence not determine any properties. It will perform all optimization with
+ * unknown sizes and hence use only the heuristic cost functions, which result in the selection
+ * of the most robust execution strategies.
+ */
+ public Optimizer() {
+ this(null, new DefaultCostEstimator());
+ }
+
+ /**
+ * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
+ * Given those statistics, the optimizer can make better choices for the execution strategies.
+ *
+ * @param stats
+ * The statistics to be used to determine the input properties.
+ */
+ public Optimizer(DataStatistics stats) {
+ this(stats, new DefaultCostEstimator());
+ }
+
+ /**
+ * Creates a new optimizer instance. The optimizer has no access to statistics about the
+ * inputs and can hence not determine any properties. It will perform all optimization with
+ * unknown sizes and hence use only the heuristic cost functions, which result in the selection
+ * of the most robust execution strategies.
+ *
+ * The optimizer uses the given cost estimator to compute the costs of the individual operations.
+ *
+ * @param estimator The cost estimator to use to cost the individual operations.
+ */
+ public Optimizer(CostEstimator estimator) {
+ this(null, estimator);
+ }
+
+ /**
+ * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
+ * Given those statistics, the optimizer can make better choices for the execution strategies.
+ *
+ * The optimizer uses the given cost estimator to compute the costs of the individual operations.
+ *
+ * @param stats
+ * The statistics to be used to determine the input properties.
+ * @param estimator
+ * The <tt>CostEstimator</tt> to use to cost the individual operations.
+ */
+ public Optimizer(DataStatistics stats, CostEstimator estimator) {
+ this.statistics = stats;
+ this.costEstimator = estimator;
+
+ // determine the default parallelism
+ this.defaultDegreeOfParallelism = GlobalConfiguration.getInteger(
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE);
+
+ if (defaultDegreeOfParallelism < 1) {
+ LOG.warn("Config value " + defaultDegreeOfParallelism + " for option "
+ + ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE + " is invalid. Ignoring and using a value of 1.");
+ this.defaultDegreeOfParallelism = 1;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Getters / Setters
+ // ------------------------------------------------------------------------
+
+ public int getDefaultDegreeOfParallelism() {
+ return defaultDegreeOfParallelism;
+ }
+
+ public void setDefaultDegreeOfParallelism(int defaultDegreeOfParallelism) {
+ if (defaultDegreeOfParallelism > 0) {
+ this.defaultDegreeOfParallelism = defaultDegreeOfParallelism;
+ } else {
+ throw new IllegalArgumentException("Default parallelism cannot be zero or negative.");
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Compilation
+ // ------------------------------------------------------------------------
+
+ /**
+ * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned
+ * and all channels have a shipping strategy assigned.
+ *
+ * For more details on the optimization phase, see the comments for
+ * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}.
+ *
+ * @param program The program to be translated.
+ * @return The optimized plan.
+ *
+ * @throws CompilerException
+ * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
+ * situation during the compilation process.
+ */
+ public OptimizedPlan compile(Plan program) throws CompilerException {
+ final OptimizerPostPass postPasser = getPostPassFromPlan(program);
+ return compile(program, postPasser);
+ }
+
+ /**
+ * Translates the given program to an OptimizedPlan. The optimized plan describes for each operator
+ * which strategy to use (such as hash join versus sort-merge join), what data exchange method to use
+ * (local pipe forward, shuffle, broadcast), what exchange mode to use (pipelined, batch),
+ * where to cache intermediate results, etc,
+ *
+ * The optimization happens in multiple phases:
+ * <ol>
+ * <li>Create optimizer dag implementation of the program.
+ *
+ * <tt>OptimizerNode</tt> representations of the PACTs, assign parallelism and compute size estimates.</li>
+ * <li>Compute interesting properties and auxiliary structures.</li>
+ * <li>Enumerate plan alternatives. This cannot be done in the same step as the interesting property computation (as
+ * opposed to the Database approaches), because we support plans that are not trees.</li>
+ * </ol>
+ *
+ * @param program The program to be translated.
+ * @param postPasser The function to be used for post passing the optimizer's plan and setting the
+ * data type specific serialization routines.
+ * @return The optimized plan.
+ *
+ * @throws CompilerException
+ * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
+ * situation during the compilation process.
+ */
+ private OptimizedPlan compile(Plan program, OptimizerPostPass postPasser) throws CompilerException {
+ if (program == null || postPasser == null) {
+ throw new NullPointerException();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Beginning compilation of program '" + program.getJobName() + '\'');
+ }
+
+ final ExecutionMode defaultDataExchangeMode = program.getExecutionConfig().getExecutionMode();
+
+ final int defaultParallelism = program.getDefaultParallelism() > 0 ?
+ program.getDefaultParallelism() : this.defaultDegreeOfParallelism;
+
+ // log the default settings
+ LOG.debug("Using a default parallelism of {}", defaultParallelism);
+ LOG.debug("Using default data exchange mode {}", defaultDataExchangeMode);
+
+ // the first step in the compilation is to create the optimizer plan representation
+ // this step does the following:
+ // 1) It creates an optimizer plan node for each operator
+ // 2) It connects them via channels
+ // 3) It looks for hints about local strategies and channel types and
+ // sets the types and strategies accordingly
+ // 4) It makes estimates about the data volume of the data sources and
+ // propagates those estimates through the plan
+
+ GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode);
+ program.accept(graphCreator);
+
+ // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
+ // each until we have only a single root node. This allows to transparently deal with the nodes with
+ // multiple outputs
+ OptimizerNode rootNode;
+ if (graphCreator.getSinks().size() == 1) {
+ rootNode = graphCreator.getSinks().get(0);
+ }
+ else if (graphCreator.getSinks().size() > 1) {
+ Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator();
+ rootNode = iter.next();
+
+ while (iter.hasNext()) {
+ rootNode = new SinkJoiner(rootNode, iter.next());
+ }
+ }
+ else {
+ throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
+ }
+
+ // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
+ // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
+ rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
+
+ // We are dealing with operator DAGs, rather than operator trees.
+ // That requires us to deviate at some points from the classical DB optimizer algorithms.
+ // This step build some auxiliary structures to help track branches and joins in the DAG
+ BranchesVisitor branchingVisitor = new BranchesVisitor();
+ rootNode.accept(branchingVisitor);
+
+ // Propagate the interesting properties top-down through the graph
+ InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
+ rootNode.accept(propsVisitor);
+
+ // perform a sanity check: the root may not have any unclosed branches
+ if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
+ throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
+ "track the re-joining of branches correctly.");
+ }
+
+ // the final step is now to generate the actual plan alternatives
+ List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);
+
+ if (bestPlan.size() != 1) {
+ throw new CompilerException("Error in compiler: more than one best plan was created!");
+ }
+
+ // check if the best plan's root is a data sink (single sink plan)
+ // if so, directly take it. if it is a sink joiner node, get its contained sinks
+ PlanNode bestPlanRoot = bestPlan.get(0);
+ List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);
+
+ if (bestPlanRoot instanceof SinkPlanNode) {
+ bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
+ } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {
+ ((SinkJoinerPlanNode) bestPlanRoot).getDataSinks(bestPlanSinks);
+ }
+
+ DeadlockPreventer dp = new DeadlockPreventer();
+ dp.resolveDeadlocks(bestPlanSinks);
+
+ // finalize the plan
+ OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program);
+
+ plan.accept(new BinaryUnionReplacer());
+
+ // post pass the plan. this is the phase where the serialization and comparator code is set
+ postPasser.postPass(plan);
+
+ return plan;
+ }
+
+ /**
+ * This function performs only the first step to the compilation process - the creation of the optimizer
+ * representation of the plan. No estimations or enumerations of alternatives are done here.
+ *
+ * @param program The plan to generate the optimizer representation for.
+ * @return The optimizer representation of the plan, as a collection of all data sinks
+ * from the plan can be traversed.
+ */
+ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
+ GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
+ program.accept(graphCreator);
+ return graphCreator.getSinks();
+ }
+
+
+ // ------------------------------------------------------------------------
+ // Miscellaneous
+ // ------------------------------------------------------------------------
+
+ private OptimizerPostPass getPostPassFromPlan(Plan program) {
+ final String className = program.getPostPassClassName();
+ if (className == null) {
+ throw new CompilerException("Optimizer Post Pass class description is null");
+ }
+ try {
+ Class<? extends OptimizerPostPass> clazz = Class.forName(className).asSubclass(OptimizerPostPass.class);
+ try {
+ return InstantiationUtil.instantiate(clazz, OptimizerPostPass.class);
+ } catch (RuntimeException rtex) {
+ // unwrap the source exception
+ if (rtex.getCause() != null) {
+ throw new CompilerException("Cannot instantiate optimizer post pass: " + rtex.getMessage(), rtex.getCause());
+ } else {
+ throw rtex;
+ }
+ }
+ }
+ catch (ClassNotFoundException cnfex) {
+ throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex);
+ }
+ catch (ClassCastException ccex) {
+ throw new CompilerException("Class '" + className + "' is not an optimizer post-pass.", ccex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
new file mode 100644
index 0000000..7880734
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.costs;
+
+import java.util.Iterator;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.EstimateProvider;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.PlanNode;
+
+/**
+ * Abstract base class for a cost estimator. Defines cost estimation methods and implements the basic work
+ * method that computes the cost of an operator by adding input shipping cost, input local cost, and
+ * driver cost.
+ */
+public abstract class CostEstimator {
+
+ public abstract void addRandomPartitioningCost(EstimateProvider estimates, Costs costs);
+
+ public abstract void addHashPartitioningCost(EstimateProvider estimates, Costs costs);
+
+ public abstract void addRangePartitionCost(EstimateProvider estimates, Costs costs);
+
+ public abstract void addBroadcastCost(EstimateProvider estimates, int replicationFactor, Costs costs);
+
+ // ------------------------------------------------------------------------
+
+ public abstract void addFileInputCost(long fileSizeInBytes, Costs costs);
+
+ public abstract void addLocalSortCost(EstimateProvider estimates, Costs costs);
+
+ public abstract void addLocalMergeCost(EstimateProvider estimates1, EstimateProvider estimates2, Costs costs, int costWeight);
+
+ public abstract void addHybridHashCosts(EstimateProvider buildSide, EstimateProvider probeSide, Costs costs, int costWeight);
+
+ public abstract void addCachedHybridHashCosts(EstimateProvider buildSide, EstimateProvider probeSide, Costs costs, int costWeight);
+
+ public abstract void addStreamedNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long bufferSize, Costs costs, int costWeight);
+
+ public abstract void addBlockNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long blockSize, Costs costs, int costWeight);
+
+ // ------------------------------------------------------------------------
+
+ public abstract void addArtificialDamCost(EstimateProvider estimates, long bufferSize, Costs costs);
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * This method computes the cost of an operator. The cost is composed of cost for input shipping,
+ * locally processing an input, and running the operator.
+ *
+ * It requires at least that all inputs are set and have a proper ship strategy set,
+ * which is not equal to <tt>NONE</tt>.
+ *
+ * @param n The node to compute the costs for.
+ */
+ public void costOperator(PlanNode n) {
+ // initialize costs objects with no costs
+ final Costs totalCosts = new Costs();
+ final long availableMemory = n.getGuaranteedAvailableMemory();
+
+ // add the shipping strategy costs
+ for (Channel channel : n.getInputs()) {
+ final Costs costs = new Costs();
+
+ // Plans that apply the same strategies, but at different points
+ // are equally expensive. For example, if a partitioning can be
+ // pushed below a Map function there is often no difference in plan
+ // costs between the pushed down version and the version that partitions
+ // after the Mapper. However, in those cases, we want the expensive
+ // strategy to appear later in the plan, as data reduction often occurs
+ // by large factors, while blowup is rare and typically by smaller fractions.
+ // We achieve this by adding a penalty to small penalty to the FORWARD strategy,
+ // weighted by the current plan depth (steps to the earliest data source).
+ // that way, later FORWARDS are more expensive than earlier forwards.
+ // Note that this only applies to the heuristic costs.
+
+ switch (channel.getShipStrategy()) {
+ case NONE:
+ throw new CompilerException(
+ "Cannot determine costs: Shipping strategy has not been set for an input.");
+ case FORWARD:
+// costs.addHeuristicNetworkCost(channel.getMaxDepth());
+ break;
+ case PARTITION_RANDOM:
+ addRandomPartitioningCost(channel, costs);
+ break;
+ case PARTITION_HASH:
+ case PARTITION_CUSTOM:
+ addHashPartitioningCost(channel, costs);
+ break;
+ case PARTITION_RANGE:
+ addRangePartitionCost(channel, costs);
+ break;
+ case BROADCAST:
+ addBroadcastCost(channel, channel.getReplicationFactor(), costs);
+ break;
+ case PARTITION_FORCED_REBALANCE:
+ addRandomPartitioningCost(channel, costs);
+ break;
+ default:
+ throw new CompilerException("Unknown shipping strategy for input: " + channel.getShipStrategy());
+ }
+
+ switch (channel.getLocalStrategy()) {
+ case NONE:
+ break;
+ case SORT:
+ case COMBININGSORT:
+ addLocalSortCost(channel, costs);
+ break;
+ default:
+ throw new CompilerException("Unsupported local strategy for input: " + channel.getLocalStrategy());
+ }
+
+ if (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE) {
+ addArtificialDamCost(channel, 0, costs);
+ }
+
+ // adjust with the cost weight factor
+ if (channel.isOnDynamicPath()) {
+ costs.multiplyWith(channel.getCostWeight());
+ }
+
+ totalCosts.addCosts(costs);
+ }
+
+ Channel firstInput = null;
+ Channel secondInput = null;
+ Costs driverCosts = new Costs();
+ int costWeight = 1;
+
+ // adjust with the cost weight factor
+ if (n.isOnDynamicPath()) {
+ costWeight = n.getCostWeight();
+ }
+
+ // get the inputs, if we have some
+ {
+ Iterator<Channel> channels = n.getInputs().iterator();
+ if (channels.hasNext()) {
+ firstInput = channels.next();
+ }
+ if (channels.hasNext()) {
+ secondInput = channels.next();
+ }
+ }
+
+ // determine the local costs
+ switch (n.getDriverStrategy()) {
+ case NONE:
+ case UNARY_NO_OP:
+ case BINARY_NO_OP:
+ case COLLECTOR_MAP:
+ case MAP:
+ case MAP_PARTITION:
+ case FLAT_MAP:
+
+ case ALL_GROUP_REDUCE:
+ case ALL_REDUCE:
+ // this operations does not do any actual grouping, since every element is in the same single group
+
+ case CO_GROUP:
+ case SORTED_GROUP_REDUCE:
+ case SORTED_REDUCE:
+ // grouping or co-grouping over sorted streams for free
+
+ case SORTED_GROUP_COMBINE:
+ // partial grouping is always local and main memory resident. we should add a relative cpu cost at some point
+
+ // partial grouping is always local and main memory resident. we should add a relative cpu cost at some point
+ case ALL_GROUP_COMBINE:
+
+ case UNION:
+ // pipelined local union is for free
+
+ break;
+ case MERGE:
+ addLocalMergeCost(firstInput, secondInput, driverCosts, costWeight);
+ break;
+ case HYBRIDHASH_BUILD_FIRST:
+ addHybridHashCosts(firstInput, secondInput, driverCosts, costWeight);
+ break;
+ case HYBRIDHASH_BUILD_SECOND:
+ addHybridHashCosts(secondInput, firstInput, driverCosts, costWeight);
+ break;
+ case HYBRIDHASH_BUILD_FIRST_CACHED:
+ addCachedHybridHashCosts(firstInput, secondInput, driverCosts, costWeight);
+ break;
+ case HYBRIDHASH_BUILD_SECOND_CACHED:
+ addCachedHybridHashCosts(secondInput, firstInput, driverCosts, costWeight);
+ break;
+ case NESTEDLOOP_BLOCKED_OUTER_FIRST:
+ addBlockNestedLoopsCosts(firstInput, secondInput, availableMemory, driverCosts, costWeight);
+ break;
+ case NESTEDLOOP_BLOCKED_OUTER_SECOND:
+ addBlockNestedLoopsCosts(secondInput, firstInput, availableMemory, driverCosts, costWeight);
+ break;
+ case NESTEDLOOP_STREAMED_OUTER_FIRST:
+ addStreamedNestedLoopsCosts(firstInput, secondInput, availableMemory, driverCosts, costWeight);
+ break;
+ case NESTEDLOOP_STREAMED_OUTER_SECOND:
+ addStreamedNestedLoopsCosts(secondInput, firstInput, availableMemory, driverCosts, costWeight);
+ break;
+ default:
+ throw new CompilerException("Unknown local strategy: " + n.getDriverStrategy().name());
+ }
+
+ totalCosts.addCosts(driverCosts);
+ n.setCosts(totalCosts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/Costs.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/Costs.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/Costs.java
new file mode 100644
index 0000000..7c854bf
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/Costs.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.costs;
+
+/**
+ * Simple class to represent the costs of an operation. The costs are currently tracking, network, I/O and CPU costs.
+ *
+ * Costs are composed of two parts of cost contributors:
+ * <ol>
+ * <li>Quantifiable costs. Those costs are used when estimates are available and track a quantifiable
+ * measure, such as the number of bytes for network or I/O</li>
+ * <li>Heuristic costs. Those costs are used when no estimates are available. They can be used to track that
+ * an operator used a special operation which is heuristically considered more expensive than another
+ * operation.</li>
+ * </ol>
+ * <p>
+ * The quantifiable costs may frequently be unknown, which is represented by a {@code -1} as a value for the unknown
+ * components of the cost. In that case, all operations' costs are unknown and hence it is not decidable which
+ * operation to favor during pruning. In that case, the heuristic costs should contain a value to make sure that
+ * operators with different strategies are comparable, even in the absence of estimates. The heuristic
+ * costs are hence the system's mechanism of realizing pruning heuristics that favor some operations over others.
+ */
+public class Costs implements Comparable<Costs>, Cloneable {
+
+ public static final double UNKNOWN = -1;
+
+ private double networkCost; // network cost, in transferred bytes
+
+ private double diskCost; // bytes to be written and read, in bytes
+
+ private double cpuCost; // CPU costs
+
+ private double heuristicNetworkCost;
+
+ private double heuristicDiskCost;
+
+ private double heuristicCpuCost;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Default constructor. Initializes all costs to 0;
+ */
+ public Costs() {
+ }
+
+ /**
+ * Creates a new costs object using the given values for the network and storage cost.
+ *
+ * @param networkCost The network cost, in bytes to be transferred.
+ * @param diskCost The cost for disk, in bytes to be written and read.
+ */
+ public Costs(double networkCost, double diskCost) {
+ setNetworkCost(networkCost);
+ setDiskCost(diskCost);
+ }
+
+ /**
+ * Creates a new costs object using the given values for the network and storage cost.
+ *
+ * @param networkCost The network cost, in bytes to be transferred.
+ * @param diskCost The cost for disk, in bytes to be written and read.
+ * @param cpuCost The cost for CPU operations.
+ */
+ public Costs(double networkCost, double diskCost, double cpuCost) {
+ setNetworkCost(networkCost);
+ setDiskCost(diskCost);
+ setCpuCost(cpuCost);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the network cost.
+ *
+ * @return The network cost, in bytes to be transferred.
+ */
+ public double getNetworkCost() {
+ return networkCost;
+ }
+
+ /**
+ * Sets the network cost for this Costs object.
+ *
+ * @param bytes
+ * The network cost to set, in bytes to be transferred.
+ */
+ public void setNetworkCost(double bytes) {
+ if (bytes == UNKNOWN || bytes >= 0) {
+ this.networkCost = bytes;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Adds the costs for network to the current network costs
+ * for this Costs object.
+ *
+ * @param bytes The network cost to add, in bytes to be transferred.
+ */
+ public void addNetworkCost(double bytes) {
+ this.networkCost = (this.networkCost < 0 || bytes < 0) ? UNKNOWN : this.networkCost + bytes;
+ }
+
+ /**
+ * Gets the costs for disk.
+ *
+ * @return The disk cost, in bytes to be written and read.
+ */
+ public double getDiskCost() {
+ return diskCost;
+ }
+
+ /**
+ * Sets the costs for disk for this Costs object.
+ *
+ * @param bytes The disk cost to set, in bytes to be written and read.
+ */
+ public void setDiskCost(double bytes) {
+ if (bytes == UNKNOWN || bytes >= 0) {
+ this.diskCost = bytes;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Adds the costs for disk to the current disk costs
+ * for this Costs object.
+ *
+ * @param bytes The disk cost to add, in bytes to be written and read.
+ */
+ public void addDiskCost(double bytes) {
+ this.diskCost =
+ (this.diskCost < 0 || bytes < 0) ? UNKNOWN : this.diskCost + bytes;
+ }
+
+ /**
+ * Gets the cost for the CPU.
+ *
+ * @return The CPU Cost.
+ */
+ public double getCpuCost() {
+ return this.cpuCost;
+ }
+
+ /**
+ * Sets the cost for the CPU.
+ *
+ * @param cost The CPU Cost.
+ */
+ public void setCpuCost(double cost) {
+ if (cost == UNKNOWN || cost >= 0) {
+ this.cpuCost = cost;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Adds the given CPU cost to the current CPU cost for this Costs object.
+ *
+ * @param cost The CPU cost to add.
+ */
+ public void addCpuCost(double cost) {
+ this.cpuCost =
+ (this.cpuCost < 0 || cost < 0) ? UNKNOWN : this.cpuCost + cost;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the heuristic network cost.
+ *
+ * @return The heuristic network cost, in bytes to be transferred.
+ */
+ public double getHeuristicNetworkCost() {
+ return this.heuristicNetworkCost;
+ }
+
+ /**
+ * Sets the heuristic network cost for this Costs object.
+ *
+ * @param cost The heuristic network cost to set, in bytes to be transferred.
+ */
+ public void setHeuristicNetworkCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicNetworkCost = cost;
+ }
+
+ /**
+ * Adds the heuristic costs for network to the current heuristic network costs
+ * for this Costs object.
+ *
+ * @param cost The heuristic network cost to add.
+ */
+ public void addHeuristicNetworkCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicNetworkCost += cost;
+ // check for overflow
+ if (this.heuristicNetworkCost < 0) {
+ this.heuristicNetworkCost = Double.MAX_VALUE;
+ }
+ }
+
+ /**
+ * Gets the heuristic costs for disk.
+ *
+ * @return The heuristic disk cost.
+ */
+ public double getHeuristicDiskCost() {
+ return this.heuristicDiskCost;
+ }
+
+ /**
+ * Sets the heuristic costs for disk for this Costs object.
+ *
+ * @param cost The heuristic disk cost to set.
+ */
+ public void setHeuristicDiskCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicDiskCost = cost;
+ }
+
+ /**
+ * Adds the heuristic costs for disk to the current heuristic disk costs
+ * for this Costs object.
+ *
+ * @param cost The heuristic disk cost to add.
+ */
+ public void addHeuristicDiskCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicDiskCost += cost;
+ // check for overflow
+ if (this.heuristicDiskCost < 0) {
+ this.heuristicDiskCost = Double.MAX_VALUE;
+ }
+ }
+
+ /**
+ * Gets the heuristic cost for the CPU.
+ *
+ * @return The heuristic CPU Cost.
+ */
+ public double getHeuristicCpuCost() {
+ return this.heuristicCpuCost;
+ }
+
+ /**
+ * Sets the heuristic cost for the CPU.
+ *
+ * @param cost The heuristic CPU Cost.
+ */
+ public void setHeuristicCpuCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicCpuCost = cost;
+ }
+
+ /**
+ * Adds the given heuristic CPU cost to the current heuristic CPU cost for this Costs object.
+ *
+ * @param cost The heuristic CPU cost to add.
+ */
+ public void addHeuristicCpuCost(double cost) {
+ if (cost <= 0) {
+ throw new IllegalArgumentException("Heuristic costs must be positive.");
+ }
+ this.heuristicCpuCost += cost;
+ // check for overflow
+ if (this.heuristicCpuCost < 0) {
+ this.heuristicCpuCost = Double.MAX_VALUE;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Adds the given costs to these costs. If for one of the different cost components (network, disk),
+ * the costs are unknown, the resulting costs will be unknown.
+ *
+ * @param other The costs to add.
+ */
+ public void addCosts(Costs other) {
+ // ---------- quantifiable costs ----------
+ if (this.networkCost == UNKNOWN || other.networkCost == UNKNOWN) {
+ this.networkCost = UNKNOWN;
+ } else {
+ this.networkCost += other.networkCost;
+ }
+
+ if (this.diskCost == UNKNOWN || other.diskCost == UNKNOWN) {
+ this.diskCost = UNKNOWN;
+ } else {
+ this.diskCost += other.diskCost;
+ }
+
+ if (this.cpuCost == UNKNOWN || other.cpuCost == UNKNOWN) {
+ this.cpuCost = UNKNOWN;
+ } else {
+ this.cpuCost += other.cpuCost;
+ }
+
+ // ---------- heuristic costs ----------
+
+ this.heuristicNetworkCost += other.heuristicNetworkCost;
+ this.heuristicDiskCost += other.heuristicDiskCost;
+ this.heuristicCpuCost += other.heuristicCpuCost;
+ }
+
+ /**
+ * Subtracts the given costs from these costs. If the given costs are unknown, then these costs are remain unchanged.
+ *
+ * @param other The costs to subtract.
+ */
+ public void subtractCosts(Costs other) {
+ if (this.networkCost != UNKNOWN && other.networkCost != UNKNOWN) {
+ this.networkCost -= other.networkCost;
+ if (this.networkCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ }
+ if (this.diskCost != UNKNOWN && other.diskCost != UNKNOWN) {
+ this.diskCost -= other.diskCost;
+ if (this.diskCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ }
+ if (this.cpuCost != UNKNOWN && other.cpuCost != UNKNOWN) {
+ this.cpuCost -= other.cpuCost;
+ if (this.cpuCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ }
+
+ // ---------- relative costs ----------
+
+ this.heuristicNetworkCost -= other.heuristicNetworkCost;
+ if (this.heuristicNetworkCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ this.heuristicDiskCost -= other.heuristicDiskCost;
+ if (this.heuristicDiskCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ this.heuristicCpuCost -= other.heuristicCpuCost;
+ if (this.heuristicCpuCost < 0) {
+ throw new IllegalArgumentException("Cannot subtract more cost then there is.");
+ }
+ }
+
+ public void multiplyWith(int factor) {
+ this.networkCost = this.networkCost < 0 ? -1 : this.networkCost * factor;
+ this.diskCost = this.diskCost < 0 ? -1 : this.diskCost * factor;
+ this.cpuCost = this.cpuCost < 0 ? -1 : this.cpuCost * factor;
+ this.heuristicNetworkCost = this.heuristicNetworkCost < 0 ? -1 : this.heuristicNetworkCost * factor;
+ this.heuristicDiskCost = this.heuristicDiskCost < 0 ? -1 : this.heuristicDiskCost * factor;
+ this.heuristicCpuCost = this.heuristicCpuCost < 0 ? -1 : this.heuristicCpuCost * factor;
+ }
+
+ public void divideBy(int factor) {
+ this.networkCost = this.networkCost < 0 ? -1 : this.networkCost / factor;
+ this.diskCost = this.diskCost < 0 ? -1 : this.diskCost / factor;
+ this.cpuCost = this.cpuCost < 0 ? -1 : this.cpuCost / factor;
+ this.heuristicNetworkCost = this.heuristicNetworkCost < 0 ? -1 : this.heuristicNetworkCost / factor;
+ this.heuristicDiskCost = this.heuristicDiskCost < 0 ? -1 : this.heuristicDiskCost / factor;
+ this.heuristicCpuCost = this.heuristicCpuCost < 0 ? -1 : this.heuristicCpuCost / factor;
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * The order of comparison is: network first, then disk, then CPU. The comparison here happens each time
+ * primarily after the heuristic costs, then after the quantifiable costs.
+ *
+ * @see java.lang.Comparable#compareTo(java.lang.Object)
+ */
+ @Override
+ public int compareTo(Costs o) {
+ // check the network cost. if we have actual costs on both, use them, otherwise use the heuristic costs.
+ if (this.networkCost != UNKNOWN && o.networkCost != UNKNOWN) {
+ if (this.networkCost != o.networkCost) {
+ return this.networkCost < o.networkCost ? -1 : 1;
+ }
+ } else if (this.heuristicNetworkCost < o.heuristicNetworkCost) {
+ return -1;
+ } else if (this.heuristicNetworkCost > o.heuristicNetworkCost) {
+ return 1;
+ }
+
+ // next, check the disk cost. again, if we have actual costs on both, use them, otherwise use the heuristic costs.
+ if (this.diskCost != UNKNOWN && o.diskCost != UNKNOWN) {
+ if (this.diskCost != o.diskCost) {
+ return this.diskCost < o.diskCost ? -1 : 1;
+ }
+ } else if (this.heuristicDiskCost < o.heuristicDiskCost) {
+ return -1;
+ } else if (this.heuristicDiskCost > o.heuristicDiskCost) {
+ return 1;
+ }
+
+ // next, check the disk cost. again, if we have actual costs on both, use them, otherwise use the heuristic costs.
+ if (this.cpuCost != UNKNOWN && o.cpuCost != UNKNOWN) {
+ return this.cpuCost < o.cpuCost ? -1 : this.cpuCost > o.cpuCost ? 1 : 0;
+ } else if (this.heuristicCpuCost < o.heuristicCpuCost) {
+ return -1;
+ } else if (this.heuristicCpuCost > o.heuristicCpuCost) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ long cpuCostBits = Double.doubleToLongBits(cpuCost);
+ long heuristicCpuCostBits = Double.doubleToLongBits(heuristicCpuCost);
+ long heuristicNetworkCostBits = Double.doubleToLongBits(heuristicNetworkCost);
+ long heuristicDiskCostBits = Double.doubleToLongBits(heuristicDiskCost);
+ long networkCostBits = Double.doubleToLongBits(networkCost);
+ long diskCostBits = Double.doubleToLongBits(diskCost);
+
+ result = prime * result + (int) (cpuCostBits ^ (cpuCostBits >>> 32));
+ result = prime * result + (int) (heuristicCpuCostBits ^ (heuristicCpuCostBits >>> 32));
+ result = prime * result + (int) (heuristicNetworkCostBits ^ (heuristicNetworkCostBits >>> 32));
+ result = prime * result + (int) (heuristicDiskCostBits ^ (heuristicDiskCostBits >>> 32));
+ result = prime * result + (int) (networkCostBits ^ (networkCostBits >>> 32));
+ result = prime * result + (int) (diskCostBits ^ (diskCostBits >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == getClass()) {
+ final Costs other = (Costs) obj;
+ return this.networkCost == other.networkCost &
+ this.diskCost == other.diskCost &
+ this.cpuCost == other.cpuCost &
+ this.heuristicNetworkCost == other.heuristicNetworkCost &
+ this.heuristicDiskCost == other.heuristicDiskCost &
+ this.heuristicCpuCost == other.heuristicCpuCost;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Costs [networkCost=" + networkCost + ", diskCost=" + diskCost +
+ ", cpuCost=" + cpuCost + ", heuristicNetworkCost=" + heuristicNetworkCost +
+ ", heuristicDiskCost=" + heuristicDiskCost + ", heuristicCpuCost=" + heuristicCpuCost + "]";
+ }
+
+ @Override
+ public Costs clone() {
+ try {
+ return (Costs) super.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e); // should never happen
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
new file mode 100644
index 0000000..543f330
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/costs/DefaultCostEstimator.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.costs;
+
+import org.apache.flink.optimizer.dag.EstimateProvider;
+
+/**
+ * A default cost estimator that has access to basic size and cardinality estimates.
+ * <p>
+ * This estimator works with actual estimates (as far as they are available) and falls back to setting
+ * relative costs, if no estimates are available. That way, the estimator makes sure that plans with
+ * different strategies are costed differently, also in the absence of estimates. The different relative
+ * costs in the absence of estimates represent this estimator's heuristic guidance towards certain strategies.
+ * <p>
+ * For robustness reasons, we always assume that the whole data is shipped during a repartition step. We deviate from
+ * the typical estimate of <code>(n - 1) / n</code> (with <i>n</i> being the number of nodes), because for a parallelism
+ * of 1, that would yield a shipping of zero bytes. While this is usually correct, the runtime scheduling may still
+ * choose to move tasks to different nodes, so that we do not know that no data is shipped.
+ */
+public class DefaultCostEstimator extends CostEstimator {
+
+ /**
+ * The case of the estimation for all relative costs. We heuristically pick a very large data volume, which
+ * will favor strategies that are less expensive on large data volumes. This is robust and
+ */
+ private static final long HEURISTIC_COST_BASE = 1000000000L;
+
+ // The numbers for the CPU effort are rather magic at the moment and should be seen rather ordinal
+
+ private static final float MATERIALIZATION_CPU_FACTOR = 1;
+
+ private static final float HASHING_CPU_FACTOR = 4;
+
+ private static final float SORTING_CPU_FACTOR = 9;
+
+
+ // --------------------------------------------------------------------------------------------
+ // Shipping Strategy Cost
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void addRandomPartitioningCost(EstimateProvider estimates, Costs costs) {
+ // conservative estimate: we need ship the whole data over the network to establish the
+ // partitioning. no disk costs.
+ final long estOutShipSize = estimates.getEstimatedOutputSize();
+ if (estOutShipSize <= 0) {
+ costs.setNetworkCost(Costs.UNKNOWN);
+ } else {
+ costs.addNetworkCost(estOutShipSize);
+ }
+ costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE);
+ }
+
+ @Override
+ public void addHashPartitioningCost(EstimateProvider estimates, Costs costs) {
+ // conservative estimate: we need ship the whole data over the network to establish the
+ // partitioning. no disk costs.
+ final long estOutShipSize = estimates.getEstimatedOutputSize();
+ if (estOutShipSize <= 0) {
+ costs.setNetworkCost(Costs.UNKNOWN);
+ } else {
+ costs.addNetworkCost(estOutShipSize);
+ }
+ costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE);
+ }
+
+ @Override
+ public void addRangePartitionCost(EstimateProvider estimates, Costs costs) {
+ final long dataSize = estimates.getEstimatedOutputSize();
+ if (dataSize > 0) {
+ // Assume sampling of 10% of the data and spilling it to disk
+ final long sampled = (long) (dataSize * 0.1f);
+ // set shipping costs
+ costs.addNetworkCost(dataSize + sampled);
+ } else {
+ costs.setNetworkCost(Costs.UNKNOWN);
+ }
+
+ // no costs known. use the same assumption as above on the heuristic costs
+ final long sampled = (long) (HEURISTIC_COST_BASE * 0.1f);
+ costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE + sampled);
+ costs.addHeuristicDiskCost(2 * sampled);
+ }
+
+ @Override
+ public void addBroadcastCost(EstimateProvider estimates, int replicationFactor, Costs costs) {
+ // if our replication factor is negative, we cannot calculate broadcast costs
+ if (replicationFactor <= 0) {
+ throw new IllegalArgumentException("The replication factor of must be larger than zero.");
+ }
+
+ if (replicationFactor > 0) {
+ // assumption: we need ship the whole data over the network to each node.
+ final long estOutShipSize = estimates.getEstimatedOutputSize();
+ if (estOutShipSize <= 0) {
+ costs.setNetworkCost(Costs.UNKNOWN);
+ } else {
+ costs.addNetworkCost(replicationFactor * estOutShipSize);
+ }
+ costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE * 10 * replicationFactor);
+ } else {
+ costs.addHeuristicNetworkCost(HEURISTIC_COST_BASE * 1000);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Local Strategy Cost
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void addFileInputCost(long fileSizeInBytes, Costs costs) {
+ if (fileSizeInBytes >= 0) {
+ costs.addDiskCost(fileSizeInBytes);
+ } else {
+ costs.setDiskCost(Costs.UNKNOWN);
+ }
+ costs.addHeuristicDiskCost(HEURISTIC_COST_BASE);
+ }
+
+ @Override
+ public void addLocalSortCost(EstimateProvider estimates, Costs costs) {
+ final long s = estimates.getEstimatedOutputSize();
+ // we assume a two phase merge sort, so all in all 2 I/O operations per block
+ if (s <= 0) {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ } else {
+ costs.addDiskCost(2 * s);
+ costs.addCpuCost((long) (s * SORTING_CPU_FACTOR));
+ }
+ costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
+ costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * SORTING_CPU_FACTOR));
+ }
+
+ @Override
+ public void addLocalMergeCost(EstimateProvider input1, EstimateProvider input2, Costs costs, int costWeight) {
+ // costs nothing. the very rarely incurred cost for a spilling block nested loops join in the
+ // presence of massively re-occurring duplicate keys is ignored, because cannot be assessed
+ }
+
+ @Override
+ public void addHybridHashCosts(EstimateProvider buildSideInput, EstimateProvider probeSideInput, Costs costs, int costWeight) {
+ long bs = buildSideInput.getEstimatedOutputSize();
+ long ps = probeSideInput.getEstimatedOutputSize();
+
+ if (bs > 0 && ps > 0) {
+ long overall = 2*bs + ps;
+ costs.addDiskCost(overall);
+ costs.addCpuCost((long) (overall * HASHING_CPU_FACTOR));
+ } else {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ }
+ costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
+ costs.addHeuristicCpuCost((long) (2 * HEURISTIC_COST_BASE * HASHING_CPU_FACTOR));
+
+ // cost weight applies to everything
+ costs.multiplyWith(costWeight);
+ }
+
+ /**
+ * Calculates the costs for the cached variant of the hybrid hash join.
+ * We are assuming by default that half of the cached hash table fit into memory.
+ */
+ @Override
+ public void addCachedHybridHashCosts(EstimateProvider buildSideInput, EstimateProvider probeSideInput, Costs costs, int costWeight) {
+ if (costWeight < 1) {
+ throw new IllegalArgumentException("The cost weight must be at least one.");
+ }
+
+ long bs = buildSideInput.getEstimatedOutputSize();
+ long ps = probeSideInput.getEstimatedOutputSize();
+
+ if (bs > 0 && ps > 0) {
+ long overall = 2*bs + costWeight*ps;
+ costs.addDiskCost(overall);
+ costs.addCpuCost((long) (overall * HASHING_CPU_FACTOR));
+ } else {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ }
+
+ // one time the build side plus cost-weight time the probe side
+ costs.addHeuristicDiskCost((1 + costWeight) * HEURISTIC_COST_BASE);
+ costs.addHeuristicCpuCost((long) ((1 + costWeight) * HEURISTIC_COST_BASE * HASHING_CPU_FACTOR));
+ }
+
+ @Override
+ public void addStreamedNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long bufferSize, Costs costs, int costWeight) {
+ long is = innerSide.getEstimatedOutputSize();
+ long oc = outerSide.getEstimatedNumRecords();
+
+ if (is > 0 && oc >= 0) {
+ // costs, if the inner side cannot be cached
+ if (is > bufferSize) {
+ costs.addDiskCost(oc * is);
+ }
+ costs.addCpuCost((long) (oc * is * MATERIALIZATION_CPU_FACTOR));
+ } else {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ }
+
+ // hack: assume 100k loops (should be expensive enough)
+ costs.addHeuristicDiskCost(HEURISTIC_COST_BASE * 100000);
+ costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * 100000 * MATERIALIZATION_CPU_FACTOR));
+ costs.multiplyWith(costWeight);
+ }
+
+ @Override
+ public void addBlockNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long blockSize, Costs costs, int costWeight) {
+ long is = innerSide.getEstimatedOutputSize();
+ long os = outerSide.getEstimatedOutputSize();
+
+ if (is > 0 && os > 0) {
+ long loops = Math.max(os / blockSize, 1);
+ costs.addDiskCost(loops * is);
+ costs.addCpuCost((long) (loops * is * MATERIALIZATION_CPU_FACTOR));
+ } else {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ }
+
+ // hack: assume 1k loops (much cheaper than the streamed variant!)
+ costs.addHeuristicDiskCost(HEURISTIC_COST_BASE * 1000);
+ costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * 1000 * MATERIALIZATION_CPU_FACTOR));
+ costs.multiplyWith(costWeight);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Damming Cost
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void addArtificialDamCost(EstimateProvider estimates, long bufferSize, Costs costs) {
+ final long s = estimates.getEstimatedOutputSize();
+ // we assume spilling and re-reading
+ if (s <= 0) {
+ costs.setDiskCost(Costs.UNKNOWN);
+ costs.setCpuCost(Costs.UNKNOWN);
+ } else {
+ costs.addDiskCost(2 * s);
+ costs.setCpuCost((long) (s * MATERIALIZATION_CPU_FACTOR));
+ }
+ costs.addHeuristicDiskCost(2 * HEURISTIC_COST_BASE);
+ costs.addHeuristicCpuCost((long) (HEURISTIC_COST_BASE * MATERIALIZATION_CPU_FACTOR));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
new file mode 100644
index 0000000..d199ae7
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
+ */
+public abstract class AbstractPartialSolutionNode extends OptimizerNode {
+
+ protected AbstractPartialSolutionNode(Operator<?> contract) {
+ super(contract);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ protected void copyEstimates(OptimizerNode node) {
+ this.estimatedNumRecords = node.estimatedNumRecords;
+ this.estimatedOutputSize = node.estimatedOutputSize;
+ }
+
+ public abstract IterationNode getIterationNode();
+
+ // --------------------------------------------------------------------------------------------
+
+ public boolean isOnDynamicPath() {
+ return true;
+ }
+
+ public void identifyDynamicPath(int costWeight) {
+ this.onDynamicPath = true;
+ this.costWeight = costWeight;
+ }
+
+ @Override
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode dataExchangeMode) {}
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // we do nothing here, because the estimates can only be copied from the iteration input
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ // no children, so nothing to compute
+ }
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new EmptySemanticProperties();
+ }
+
+ @Override
+ protected void readStubAnnotations() {}
+
+ @Override
+ public void accept(Visitor<OptimizerNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
new file mode 100644
index 0000000..068799e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
@@ -0,0 +1,308 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.Union;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.BinaryUnionOpDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+/**
+ * The Optimizer representation of a binary <i>Union</i>.
+ */
+public class BinaryUnionNode extends TwoInputNode {
+
+ private Set<RequestedGlobalProperties> channelProps;
+
+ public BinaryUnionNode(Union<?> union){
+ super(union);
+ }
+
+ @Override
+ public String getName() {
+ return "Union";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
+ long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
+ this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 + card2;
+
+ long size1 = getFirstPredecessorNode().getEstimatedOutputSize();
+ long size2 = getSecondPredecessorNode().getEstimatedOutputSize();
+ this.estimatedOutputSize = (size1 < 0 || size2 < 0) ? -1 : size1 + size2;
+ }
+
+ @Override
+ public void computeUnionOfInterestingPropertiesFromSuccessors() {
+ super.computeUnionOfInterestingPropertiesFromSuccessors();
+ // clear all local properties, as they are destroyed anyways
+ getInterestingProperties().getLocalProperties().clear();
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ final InterestingProperties props = getInterestingProperties();
+
+ // if no other properties exist, add the pruned trivials back
+ if (props.getGlobalProperties().isEmpty()) {
+ props.addGlobalProperties(new RequestedGlobalProperties());
+ }
+ props.addLocalProperties(new RequestedLocalProperties());
+ this.input1.setInterestingProperties(props.clone());
+ this.input2.setInterestingProperties(props.clone());
+
+ this.channelProps = props.getGlobalProperties();
+ }
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ // check if we have a cached version
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ }
+
+ // step down to all producer nodes and calculate alternative plans
+ final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
+ final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
+
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
+ if (broadcastConnections != null && broadcastConnections.size() > 0) {
+ throw new CompilerException("Found BroadcastVariables on a Union operation");
+ }
+
+ final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
+
+ final List<Set<? extends NamedChannel>> broadcastPlanChannels = Collections.emptyList();
+
+ final BinaryUnionOpDescriptor operator = new BinaryUnionOpDescriptor();
+ final RequestedLocalProperties noLocalProps = new RequestedLocalProperties();
+
+ final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
+ final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
+
+ final int dop = getParallelism();
+ final int inDop1 = getFirstPredecessorNode().getParallelism();
+ final int inDop2 = getSecondPredecessorNode().getParallelism();
+
+ final boolean dopChange1 = dop != inDop1;
+ final boolean dopChange2 = dop != inDop2;
+
+ final boolean input1breakPipeline = this.input1.isBreakingPipeline();
+ final boolean input2breakPipeline = this.input2.isBreakingPipeline();
+
+ // enumerate all pairwise combination of the children's plans together with
+ // all possible operator strategy combination
+
+ // create all candidates
+ for (PlanNode child1 : subPlans1) {
+ for (PlanNode child2 : subPlans2) {
+
+ // check that the children go together. that is the case if they build upon the same
+ // candidate at the joined branch plan.
+ if (!areBranchCompatible(child1, child2)) {
+ continue;
+ }
+
+ for (RequestedGlobalProperties igps: this.channelProps) {
+ // create a candidate channel for the first input. mark it cached, if the connection says so
+ Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
+ if (this.input1.getShipStrategy() == null) {
+ // free to choose the ship strategy
+ igps.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
+
+ // if the DOP changed, make sure that we cancel out properties, unless the
+ // ship strategy preserves/establishes them even under changing DOPs
+ if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
+ c1.getGlobalProperties().reset();
+ }
+ }
+ else {
+ // ship strategy fixed by compiler hint
+ ShipStrategyType shipStrategy = this.input1.getShipStrategy();
+ DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipStrategy, input1breakPipeline);
+ if (this.keys1 != null) {
+ c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList(), exMode);
+ } else {
+ c1.setShipStrategy(this.input1.getShipStrategy(), exMode);
+ }
+
+ if (dopChange1) {
+ c1.adjustGlobalPropertiesForFullParallelismChange();
+ }
+ }
+
+ // create a candidate channel for the first input. mark it cached, if the connection says so
+ Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
+ if (this.input2.getShipStrategy() == null) {
+ // free to choose the ship strategy
+ igps.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
+
+ // if the DOP changed, make sure that we cancel out properties, unless the
+ // ship strategy preserves/establishes them even under changing DOPs
+ if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
+ c2.getGlobalProperties().reset();
+ }
+ } else {
+ // ship strategy fixed by compiler hint
+ ShipStrategyType shipStrategy = this.input2.getShipStrategy();
+ DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipStrategy, input2breakPipeline);
+ if (this.keys2 != null) {
+ c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList(), exMode);
+ } else {
+ c2.setShipStrategy(this.input2.getShipStrategy(), exMode);
+ }
+
+ if (dopChange2) {
+ c2.adjustGlobalPropertiesForFullParallelismChange();
+ }
+ }
+
+ // get the global properties and clear unique fields (not preserved anyways during the union)
+ GlobalProperties p1 = c1.getGlobalProperties();
+ GlobalProperties p2 = c2.getGlobalProperties();
+ p1.clearUniqueFieldCombinations();
+ p2.clearUniqueFieldCombinations();
+
+ // adjust the partitioning, if they exist but are not equal. this may happen when both channels have a
+ // partitioning that fulfills the requirements, but both are incompatible. For example may a property requirement
+ // be ANY_PARTITIONING on fields (0) and one channel is range partitioned on that field, the other is hash
+ // partitioned on that field.
+ if (!igps.isTrivial() && !(p1.equals(p2))) {
+ if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() != ShipStrategyType.FORWARD) {
+ // adjust c2 to c1
+ c2 = c2.clone();
+ p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
+ }
+ else if (c2.getShipStrategy() == ShipStrategyType.FORWARD && c1.getShipStrategy() != ShipStrategyType.FORWARD) {
+ // adjust c1 to c2
+ c1 = c1.clone();
+ p2.parameterizeChannel(c1,dopChange1, input1Mode, input1breakPipeline);
+ }
+ else if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() == ShipStrategyType.FORWARD) {
+ boolean adjustC1 = c1.getEstimatedOutputSize() <= 0 || c2.getEstimatedOutputSize() <= 0 ||
+ c1.getEstimatedOutputSize() <= c2.getEstimatedOutputSize();
+ if (adjustC1) {
+ c2 = c2.clone();
+ p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
+ } else {
+ c1 = c1.clone();
+ p2.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
+ }
+ } else {
+ // this should never happen, as it implies both realize a different strategy, which is
+ // excluded by the check that the required strategies must match
+ throw new CompilerException("Bug in Plan Enumeration for Union Node.");
+ }
+ }
+
+
+ instantiate(operator, c1, c2, broadcastPlanChannels, outputPlans, estimator, igps, igps, noLocalProps, noLocalProps);
+ }
+ }
+ }
+
+ // cost and prune the plans
+ for (PlanNode node : outputPlans) {
+ estimator.costOperator(node);
+ }
+ prunePlanAlternatives(outputPlans);
+ outputPlans.trimToSize();
+
+ this.cachedPlans = outputPlans;
+ return outputPlans;
+ }
+
+ @Override
+ protected void readStubAnnotations() {}
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new UnionSemanticProperties();
+ }
+
+ @Override
+ public void computeOutputEstimates(DataStatistics statistics) {
+ OptimizerNode in1 = getFirstPredecessorNode();
+ OptimizerNode in2 = getSecondPredecessorNode();
+
+ this.estimatedNumRecords = in1.estimatedNumRecords > 0 && in2.estimatedNumRecords > 0 ?
+ in1.estimatedNumRecords + in2.estimatedNumRecords : -1;
+ this.estimatedOutputSize = in1.estimatedOutputSize > 0 && in2.estimatedOutputSize > 0 ?
+ in1.estimatedOutputSize + in2.estimatedOutputSize : -1;
+ }
+
+ public static class UnionSemanticProperties implements SemanticProperties {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FieldSet getForwardingTargetFields(int input, int sourceField) {
+ if (input != 0 && input != 1) {
+ throw new IndexOutOfBoundsException("Invalid input index for binary union node.");
+ }
+
+ return new FieldSet(sourceField);
+ }
+
+ @Override
+ public int getForwardingSourceField(int input, int targetField) {
+ if (input != 0 && input != 1) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ return targetField;
+ }
+
+ @Override
+ public FieldSet getReadFields(int input) {
+ if (input != 0 && input != 1) {
+ throw new IndexOutOfBoundsException();
+ }
+
+ return FieldSet.EMPTY_SET;
+ }
+
+ }
+}
[17/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
new file mode 100644
index 0000000..cc12bb8
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
@@ -0,0 +1,518 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport;
+import org.apache.flink.optimizer.util.NoOpUnaryUdfOp;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+
+import com.google.common.collect.Sets;
+
+/**
+ * A node in the optimizer's program representation for an operation with a single input.
+ *
+ * This class contains all the generic logic for handling branching flows, as well as to
+ * enumerate candidate execution plans. The subclasses for specific operators simply add logic
+ * for cost estimates and specify possible strategies for their execution.
+ */
+public abstract class SingleInputNode extends OptimizerNode {
+
+ protected final FieldSet keys; // The set of key fields
+
+ protected DagConnection inConn; // the input of the node
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new node with a single input for the optimizer plan.
+ *
+ * @param programOperator The PACT that the node represents.
+ */
+ protected SingleInputNode(SingleInputOperator<?, ?, ?> programOperator) {
+ super(programOperator);
+
+ int[] k = programOperator.getKeyColumns(0);
+ this.keys = k == null || k.length == 0 ? null : new FieldSet(k);
+ }
+
+ protected SingleInputNode(FieldSet keys) {
+ super(NoOpUnaryUdfOp.INSTANCE);
+ this.keys = keys;
+ }
+
+ protected SingleInputNode() {
+ super(NoOpUnaryUdfOp.INSTANCE);
+ this.keys = null;
+ }
+
+ protected SingleInputNode(SingleInputNode toCopy) {
+ super(toCopy);
+
+ this.keys = toCopy.keys;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public SingleInputOperator<?, ?, ?> getOperator() {
+ return (SingleInputOperator<?, ?, ?>) super.getOperator();
+ }
+
+ /**
+ * Gets the input of this operator.
+ *
+ * @return The input.
+ */
+ public DagConnection getIncomingConnection() {
+ return this.inConn;
+ }
+
+ /**
+ * Sets the connection through which this node receives its input.
+ *
+ * @param inConn The input connection to set.
+ */
+ public void setIncomingConnection(DagConnection inConn) {
+ this.inConn = inConn;
+ }
+
+ /**
+ * Gets the predecessor of this node.
+ *
+ * @return The predecessor of this node.
+ */
+ public OptimizerNode getPredecessorNode() {
+ if (this.inConn != null) {
+ return this.inConn.getSource();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.singletonList(this.inConn);
+ }
+
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return getOperator().getSemanticProperties();
+ }
+
+
+ @Override
+ public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
+ throws CompilerException
+ {
+ // see if an internal hint dictates the strategy to use
+ final Configuration conf = getOperator().getParameters();
+ final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
+ final ShipStrategyType preSet;
+
+ if (shipStrategy != null) {
+ if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
+ preSet = ShipStrategyType.PARTITION_HASH;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
+ preSet = ShipStrategyType.PARTITION_RANGE;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
+ preSet = ShipStrategyType.FORWARD;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
+ preSet = ShipStrategyType.PARTITION_RANDOM;
+ } else {
+ throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
+ }
+ } else {
+ preSet = null;
+ }
+
+ // get the predecessor node
+ Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
+
+ OptimizerNode pred;
+ DagConnection conn;
+ if (children == null) {
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
+ } else {
+ pred = contractToNode.get(children);
+ conn = new DagConnection(pred, this, defaultExchangeMode);
+ if (preSet != null) {
+ conn.setShipStrategy(preSet);
+ }
+ }
+
+ // create the connection and add it
+ setIncomingConnection(conn);
+ pred.addOutgoingConnection(conn);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Properties and Optimization
+ // --------------------------------------------------------------------------------------------
+
+ protected abstract List<OperatorDescriptorSingle> getPossibleProperties();
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ // get what we inherit and what is preserved by our user code
+ final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
+
+ // add all properties relevant to this node
+ for (OperatorDescriptorSingle dps : getPossibleProperties()) {
+ for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
+
+ if (gp.getPartitioning().isPartitionedOnKey()) {
+ // make sure that among the same partitioning types, we do not push anything down that has fewer key fields
+
+ for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
+ if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
+ props.getGlobalProperties().remove(contained);
+ break;
+ }
+ }
+ }
+
+ props.addGlobalProperties(gp);
+ }
+
+ for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
+ props.addLocalProperties(lp);
+ }
+ }
+ this.inConn.setInterestingProperties(props);
+
+ for (DagConnection conn : getBroadcastConnections()) {
+ conn.setInterestingProperties(new InterestingProperties());
+ }
+ }
+
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ // check if we have a cached version
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ }
+
+ boolean childrenSkippedDueToReplicatedInput = false;
+
+ // calculate alternative sub-plans for predecessor
+ final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
+ final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties();
+
+ // calculate alternative sub-plans for broadcast inputs
+ final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
+ List<String> broadcastConnectionNames = getBroadcastConnectionNames();
+
+ for (int i = 0; i < broadcastConnections.size(); i++ ) {
+ DagConnection broadcastConnection = broadcastConnections.get(i);
+ String broadcastConnectionName = broadcastConnectionNames.get(i);
+ List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
+
+ // wrap the plan candidates in named channels
+ HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
+ for (PlanNode plan: broadcastPlanCandidates) {
+ NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
+ DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(),
+ ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
+ c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
+ broadcastChannels.add(c);
+ }
+ broadcastPlanChannels.add(broadcastChannels);
+ }
+
+ final RequestedGlobalProperties[] allValidGlobals;
+ {
+ Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>();
+ for (OperatorDescriptorSingle ods : getPossibleProperties()) {
+ pairs.addAll(ods.getPossibleGlobalProperties());
+ }
+ allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]);
+ }
+ final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
+
+ final ExecutionMode executionMode = this.inConn.getDataExchangeMode();
+
+ final int dop = getParallelism();
+ final int inDop = getPredecessorNode().getParallelism();
+ final boolean dopChange = inDop != dop;
+
+ final boolean breaksPipeline = this.inConn.isBreakingPipeline();
+
+ // create all candidates
+ for (PlanNode child : subPlans) {
+
+ if (child.getGlobalProperties().isFullyReplicated()) {
+ // fully replicated input is always locally forwarded if DOP is not changed
+ if (dopChange) {
+ // can not continue with this child
+ childrenSkippedDueToReplicatedInput = true;
+ continue;
+ } else {
+ this.inConn.setShipStrategy(ShipStrategyType.FORWARD);
+ }
+ }
+
+ if (this.inConn.getShipStrategy() == null) {
+ // pick the strategy ourselves
+ for (RequestedGlobalProperties igps: intGlobal) {
+ final Channel c = new Channel(child, this.inConn.getMaterializationMode());
+ igps.parameterizeChannel(c, dopChange, executionMode, breaksPipeline);
+
+ // if the DOP changed, make sure that we cancel out properties, unless the
+ // ship strategy preserves/establishes them even under changing DOPs
+ if (dopChange && !c.getShipStrategy().isNetworkStrategy()) {
+ c.getGlobalProperties().reset();
+ }
+
+ // check whether we meet any of the accepted properties
+ // we may remove this check, when we do a check to not inherit
+ // requested global properties that are incompatible with all possible
+ // requested properties
+ for (RequestedGlobalProperties rgps: allValidGlobals) {
+ if (rgps.isMetBy(c.getGlobalProperties())) {
+ c.setRequiredGlobalProps(rgps);
+ addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
+ break;
+ }
+ }
+ }
+ } else {
+ // hint fixed the strategy
+ final Channel c = new Channel(child, this.inConn.getMaterializationMode());
+ final ShipStrategyType shipStrategy = this.inConn.getShipStrategy();
+ final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);
+
+ if (this.keys != null) {
+ c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode);
+ } else {
+ c.setShipStrategy(shipStrategy, exMode);
+ }
+
+ if (dopChange) {
+ c.adjustGlobalPropertiesForFullParallelismChange();
+ }
+
+ // check whether we meet any of the accepted properties
+ for (RequestedGlobalProperties rgps: allValidGlobals) {
+ if (rgps.isMetBy(c.getGlobalProperties())) {
+ addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
+ break;
+ }
+ }
+ }
+ }
+
+ if(outputPlans.isEmpty()) {
+ if(childrenSkippedDueToReplicatedInput) {
+ throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input.");
+ } else {
+ throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
+ }
+ }
+
+ // cost and prune the plans
+ for (PlanNode node : outputPlans) {
+ estimator.costOperator(node);
+ }
+ prunePlanAlternatives(outputPlans);
+ outputPlans.trimToSize();
+
+ this.cachedPlans = outputPlans;
+ return outputPlans;
+ }
+
+ protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
+ List<PlanNode> target, CostEstimator estimator)
+ {
+ for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
+ final Channel in = template.clone();
+ ilp.parameterizeChannel(in);
+
+ // instantiate a candidate, if the instantiated local properties meet one possible local property set
+ outer:
+ for (OperatorDescriptorSingle dps: getPossibleProperties()) {
+ for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
+ if (ilps.isMetBy(in.getLocalProperties())) {
+ in.setRequiredLocalProps(ilps);
+ instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
+ break outer;
+ }
+ }
+ }
+ }
+ }
+
+ protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
+ List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
+ {
+ final PlanNode inputSource = in.getSource();
+
+ for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
+
+ boolean validCombination = true;
+ boolean requiresPipelinebreaker = false;
+
+ // check whether the broadcast inputs use the same plan candidate at the branching point
+ for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
+ NamedChannel nc = broadcastChannelsCombination.get(i);
+ PlanNode bcSource = nc.getSource();
+
+ // check branch compatibility against input
+ if (!areBranchCompatible(bcSource, inputSource)) {
+ validCombination = false;
+ break;
+ }
+
+ // check branch compatibility against all other broadcast variables
+ for (int k = 0; k < i; k++) {
+ PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
+
+ if (!areBranchCompatible(bcSource, otherBcSource)) {
+ validCombination = false;
+ break;
+ }
+ }
+
+ // check if there is a common predecessor and whether there is a dam on the way to all common predecessors
+ if (this.hereJoinedBranches != null) {
+ for (OptimizerNode brancher : this.hereJoinedBranches) {
+ PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
+
+ if (candAtBrancher == null) {
+ // closed branch between two broadcast variables
+ continue;
+ }
+
+ SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
+ if (res == NOT_FOUND) {
+ throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
+ } else if (res == FOUND_SOURCE) {
+ requiresPipelinebreaker = true;
+ break;
+ } else if (res == FOUND_SOURCE_AND_DAM) {
+ // good
+ } else {
+ throw new CompilerException();
+ }
+ }
+ }
+ }
+
+ if (!validCombination) {
+ continue;
+ }
+
+ if (requiresPipelinebreaker) {
+ in.setTempMode(in.getTempMode().makePipelineBreaker());
+ }
+
+ final SingleInputPlanNode node = dps.instantiate(in, this);
+ node.setBroadcastInputs(broadcastChannelsCombination);
+
+ // compute how the strategy affects the properties
+ GlobalProperties gProps = in.getGlobalProperties().clone();
+ LocalProperties lProps = in.getLocalProperties().clone();
+ gProps = dps.computeGlobalProperties(gProps);
+ lProps = dps.computeLocalProperties(lProps);
+
+ SemanticProperties props = this.getSemanticProperties();
+ // filter by the user code field copies
+ gProps = gProps.filterBySemanticProperties(props, 0);
+ lProps = lProps.filterBySemanticProperties(props, 0);
+
+ // apply
+ node.initProperties(gProps, lProps);
+ node.updatePropertiesWithUniqueSets(getUniqueFields());
+ target.add(node);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Branch Handling
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ addClosedBranches(getPredecessorNode().closedBranchingNodes);
+ List<UnclosedBranchDescriptor> fromInput = getPredecessorNode().getBranchesForParent(this.inConn);
+
+ // handle the data flow branching for the broadcast inputs
+ List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(fromInput);
+
+ this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void accept(Visitor<OptimizerNode> visitor) {
+ if (visitor.preVisit(this)) {
+ if (getPredecessorNode() != null) {
+ getPredecessorNode().accept(visitor);
+ } else {
+ throw new CompilerException();
+ }
+ for (DagConnection connection : getBroadcastConnections()) {
+ connection.getSource().accept(visitor);
+ }
+ visitor.postVisit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
new file mode 100644
index 0000000..40725ba
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.operators.UtilSinkJoinOpDescriptor;
+import org.apache.flink.optimizer.util.NoOpBinaryUdfOp;
+import org.apache.flink.types.Nothing;
+
+/**
+ * This class represents a utility node that is not part of the actual plan.
+ * It is used for plans with multiple data sinks to transform it into a plan with
+ * a single root node. That way, the code that makes sure no costs are double-counted and that
+ * candidate selection works correctly with nodes that have multiple outputs is transparently reused.
+ */
+public class SinkJoiner extends TwoInputNode {
+
+ public SinkJoiner(OptimizerNode input1, OptimizerNode input2) {
+ super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));
+
+ DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED);
+ DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED);
+
+ this.input1 = conn1;
+ this.input2 = conn2;
+
+ setDegreeOfParallelism(1);
+ }
+
+ @Override
+ public String getName() {
+ return "Internal Utility Node";
+ }
+
+ @Override
+ public List<DagConnection> getOutgoingConnections() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
+ addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
+
+ List<UnclosedBranchDescriptor> pred1branches = getFirstPredecessorNode().openBranches;
+ List<UnclosedBranchDescriptor> pred2branches = getSecondPredecessorNode().openBranches;
+
+ // if the predecessors do not have branches, then we have multiple sinks that do not originate from
+ // a common data flow.
+ if (pred1branches == null || pred1branches.isEmpty()) {
+
+ this.openBranches = (pred2branches == null || pred2branches.isEmpty()) ?
+ Collections.<UnclosedBranchDescriptor>emptyList() : // both empty - disconnected flow
+ pred2branches;
+ }
+ else if (pred2branches == null || pred2branches.isEmpty()) {
+ this.openBranches = pred1branches;
+ }
+ else {
+ // copy the lists and merge
+ List<UnclosedBranchDescriptor> result1 = new ArrayList<UnclosedBranchDescriptor>(pred1branches);
+ List<UnclosedBranchDescriptor> result2 = new ArrayList<UnclosedBranchDescriptor>(pred2branches);
+
+ ArrayList<UnclosedBranchDescriptor> result = new ArrayList<UnclosedBranchDescriptor>();
+ mergeLists(result1, result2, result, false);
+
+ this.openBranches = result.isEmpty() ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
+ }
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return Collections.<OperatorDescriptorDual>singletonList(new UtilSinkJoinOpDescriptor());
+ }
+
+ @Override
+ public void computeOutputEstimates(DataStatistics statistics) {
+ // nothing to be done here
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // no estimates needed at this point
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
new file mode 100644
index 0000000..1292cf5
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+
+/**
+ * The optimizer's internal representation of the solution set of a workset iteration.
+ */
+public class SolutionSetNode extends AbstractPartialSolutionNode {
+
+ private final WorksetIterationNode iterationNode;
+
+
+ public SolutionSetNode(SolutionSetPlaceHolder<?> psph, WorksetIterationNode iterationNode) {
+ super(psph);
+ this.iterationNode = iterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
+ this.cachedPlans = Collections.<PlanNode>singletonList(new SolutionSetPlanNode(this, "SolutionSet("+this.getOperator().getName()+")", gProps, lProps, initialInput));
+ }
+
+ public SolutionSetPlanNode getCurrentSolutionSetPlanNode() {
+ if (this.cachedPlans != null) {
+ return (SolutionSetPlanNode) this.cachedPlans.get(0);
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ public WorksetIterationNode getIterationNode() {
+ return this.iterationNode;
+ }
+
+ @Override
+ public void computeOutputEstimates(DataStatistics statistics) {
+ copyEstimates(this.iterationNode.getInitialSolutionSetPredecessorNode());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the contract object for this data source node.
+ *
+ * @return The contract.
+ */
+ @Override
+ public SolutionSetPlaceHolder<?> getOperator() {
+ return (SolutionSetPlaceHolder<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Solution Set";
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ DagConnection solutionSetInput = this.iterationNode.getFirstIncomingConnection();
+ OptimizerNode solutionSetSource = solutionSetInput.getSource();
+
+ addClosedBranches(solutionSetSource.closedBranchingNodes);
+ List<UnclosedBranchDescriptor> fromInput = solutionSetSource.getBranchesForParent(solutionSetInput);
+ this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
new file mode 100644
index 0000000..83bc39a
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The optimizer's internal representation of a <i>SortPartition</i> operator node.
+ */
+public class SortPartitionNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ public SortPartitionNode(SortPartitionOperatorBase<?> operator) {
+ super(operator);
+
+ OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering());
+ this.possibleProperties = Collections.singletonList(descr);
+ }
+
+ @Override
+ public SortPartitionOperatorBase<?> getOperator() {
+ return (SortPartitionOperatorBase<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Sort-Partition";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // sorting does not change the number of records
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class SortPartitionDescriptor extends OperatorDescriptorSingle {
+
+ private Ordering partitionOrder;
+
+ public SortPartitionDescriptor(Ordering partitionOrder) {
+ this.partitionOrder = partitionOrder;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.UNARY_NO_OP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Sort-Partition", in, DriverStrategy.UNARY_NO_OP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ // sort partition does not require any global property
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ // set partition order as required local property
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setOrdering(this.partitionOrder);
+
+ return Collections.singletonList(rlp);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ // sort partition is a no-operation operation, such that all global properties are preserved.
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ // sort partition is a no-operation operation, such that all global properties are preserved.
+ return lProps;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TempMode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
new file mode 100644
index 0000000..0d1dfc9
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TempMode.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+/**
+ * Enumeration to indicate the mode of temporarily materializing the data that flows across a connection.
+ * Introducing such an artificial dam is sometimes necessary to avoid that a certain data flows deadlock
+ * themselves, or as a cache to replay an intermediate result.
+ */
+public enum TempMode {
+
+ NONE(false, false),
+ PIPELINE_BREAKER(false, true),
+ CACHED(true, false),
+ CACHING_PIPELINE_BREAKER(true, true);
+
+ // --------------------------------------------------------------------------------------------
+
+ private final boolean cached;
+
+ private final boolean breaksPipeline;
+
+
+ private TempMode(boolean cached, boolean breaksPipeline) {
+ this.cached = cached;
+ this.breaksPipeline = breaksPipeline;
+ }
+
+ public boolean isCached() {
+ return cached;
+ }
+
+ public boolean breaksPipeline() {
+ return breaksPipeline;
+ }
+
+ public TempMode makePipelineBreaker() {
+ if (this == NONE) {
+ return PIPELINE_BREAKER;
+ } else if (this == CACHED) {
+ return CACHING_PIPELINE_BREAKER;
+ } else {
+ return this;
+ }
+ }
+
+ public TempMode makeCached() {
+ if (this == NONE) {
+ return CACHED;
+ } else if (this == PIPELINE_BREAKER) {
+ return CACHING_PIPELINE_BREAKER;
+ } else {
+ return this;
+ }
+ }
+
+
+ public TempMode makeNonCached() {
+ if (this == CACHED) {
+ return NONE;
+ } else if (this == CACHING_PIPELINE_BREAKER) {
+ return PIPELINE_BREAKER;
+ } else {
+ return this;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
new file mode 100644
index 0000000..39da165
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/TwoInputNode.java
@@ -0,0 +1,747 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.DualInputOperator;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual.GlobalPropertiesPair;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual.LocalPropertiesPair;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+
+import com.google.common.collect.Sets;
+
+/**
+ * A node in the optimizer plan that represents a PACT with a two different inputs, such as MATCH or CROSS.
+ * The two inputs are not substitutable in their sides.
+ */
+public abstract class TwoInputNode extends OptimizerNode {
+
+ protected final FieldList keys1; // The set of key fields for the first input
+
+ protected final FieldList keys2; // The set of key fields for the second input
+
+ protected DagConnection input1; // The first input edge
+
+ protected DagConnection input2; // The second input edge
+
+ private List<OperatorDescriptorDual> cachedDescriptors;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new node with a single input for the optimizer plan.
+ *
+ * @param pactContract
+ * The PACT that the node represents.
+ */
+ public TwoInputNode(DualInputOperator<?, ?, ?, ?> pactContract) {
+ super(pactContract);
+
+ int[] k1 = pactContract.getKeyColumns(0);
+ int[] k2 = pactContract.getKeyColumns(1);
+
+ this.keys1 = k1 == null || k1.length == 0 ? null : new FieldList(k1);
+ this.keys2 = k2 == null || k2.length == 0 ? null : new FieldList(k2);
+
+ if (this.keys1 != null) {
+ if (this.keys2 != null) {
+ if (this.keys1.size() != this.keys2.size()) {
+ throw new CompilerException("Unequal number of key fields on the two inputs.");
+ }
+ } else {
+ throw new CompilerException("Keys are set on first input, but not on second.");
+ }
+ } else if (this.keys2 != null) {
+ throw new CompilerException("Keys are set on second input, but not on first.");
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public DualInputOperator<?, ?, ?, ?> getOperator() {
+ return (DualInputOperator<?, ?, ?, ?>) super.getOperator();
+ }
+
+ /**
+ * Gets the <tt>PactConnection</tt> through which this node receives its <i>first</i> input.
+ *
+ * @return The first input connection.
+ */
+ public DagConnection getFirstIncomingConnection() {
+ return this.input1;
+ }
+
+ /**
+ * Gets the <tt>PactConnection</tt> through which this node receives its <i>second</i> input.
+ *
+ * @return The second input connection.
+ */
+ public DagConnection getSecondIncomingConnection() {
+ return this.input2;
+ }
+
+ public OptimizerNode getFirstPredecessorNode() {
+ if(this.input1 != null) {
+ return this.input1.getSource();
+ } else {
+ return null;
+ }
+ }
+
+ public OptimizerNode getSecondPredecessorNode() {
+ if(this.input2 != null) {
+ return this.input2.getSource();
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public List<DagConnection> getIncomingConnections() {
+ ArrayList<DagConnection> inputs = new ArrayList<DagConnection>(2);
+ inputs.add(input1);
+ inputs.add(input2);
+ return inputs;
+ }
+
+
+ @Override
+ public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
+ // see if there is a hint that dictates which shipping strategy to use for BOTH inputs
+ final Configuration conf = getOperator().getParameters();
+ ShipStrategyType preSet1 = null;
+ ShipStrategyType preSet2 = null;
+
+ String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
+ if (shipStrategy != null) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ preSet1 = preSet2 = ShipStrategyType.FORWARD;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ preSet1 = preSet2 = ShipStrategyType.BROADCAST;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
+ preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
+ } else {
+ throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
+ }
+ }
+
+ // see if there is a hint that dictates which shipping strategy to use for the FIRST input
+ shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
+ if (shipStrategy != null) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ preSet1 = ShipStrategyType.FORWARD;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ preSet1 = ShipStrategyType.BROADCAST;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ preSet1 = ShipStrategyType.PARTITION_HASH;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ preSet1 = ShipStrategyType.PARTITION_RANGE;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
+ preSet1 = ShipStrategyType.PARTITION_RANDOM;
+ } else {
+ throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
+ }
+ }
+
+ // see if there is a hint that dictates which shipping strategy to use for the SECOND input
+ shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
+ if (shipStrategy != null) {
+ if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
+ preSet2 = ShipStrategyType.FORWARD;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
+ preSet2 = ShipStrategyType.BROADCAST;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
+ preSet2 = ShipStrategyType.PARTITION_HASH;
+ } else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
+ preSet2 = ShipStrategyType.PARTITION_RANGE;
+ } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
+ preSet2 = ShipStrategyType.PARTITION_RANDOM;
+ } else {
+ throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
+ }
+ }
+
+ // get the predecessors
+ DualInputOperator<?, ?, ?, ?> contr = getOperator();
+
+ Operator<?> leftPred = contr.getFirstInput();
+ Operator<?> rightPred = contr.getSecondInput();
+
+ OptimizerNode pred1;
+ DagConnection conn1;
+ if (leftPred == null) {
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
+ } else {
+ pred1 = contractToNode.get(leftPred);
+ conn1 = new DagConnection(pred1, this, defaultExecutionMode);
+ if (preSet1 != null) {
+ conn1.setShipStrategy(preSet1);
+ }
+ }
+
+ // create the connection and add it
+ this.input1 = conn1;
+ pred1.addOutgoingConnection(conn1);
+
+ OptimizerNode pred2;
+ DagConnection conn2;
+ if (rightPred == null) {
+ throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
+ } else {
+ pred2 = contractToNode.get(rightPred);
+ conn2 = new DagConnection(pred2, this, defaultExecutionMode);
+ if (preSet2 != null) {
+ conn2.setShipStrategy(preSet2);
+ }
+ }
+
+ // create the connection and add it
+ this.input2 = conn2;
+ pred2.addOutgoingConnection(conn2);
+ }
+
+ protected abstract List<OperatorDescriptorDual> getPossibleProperties();
+
+ private List<OperatorDescriptorDual> getProperties() {
+ if (this.cachedDescriptors == null) {
+ this.cachedDescriptors = getPossibleProperties();
+ }
+ return this.cachedDescriptors;
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ // get what we inherit and what is preserved by our user code
+ final InterestingProperties props1 = getInterestingProperties().filterByCodeAnnotations(this, 0);
+ final InterestingProperties props2 = getInterestingProperties().filterByCodeAnnotations(this, 1);
+
+ // add all properties relevant to this node
+ for (OperatorDescriptorDual dpd : getProperties()) {
+ for (GlobalPropertiesPair gp : dpd.getPossibleGlobalProperties()) {
+ // input 1
+ props1.addGlobalProperties(gp.getProperties1());
+
+ // input 2
+ props2.addGlobalProperties(gp.getProperties2());
+ }
+ for (LocalPropertiesPair lp : dpd.getPossibleLocalProperties()) {
+ // input 1
+ props1.addLocalProperties(lp.getProperties1());
+
+ // input 2
+ props2.addLocalProperties(lp.getProperties2());
+ }
+ }
+ this.input1.setInterestingProperties(props1);
+ this.input2.setInterestingProperties(props2);
+
+ for (DagConnection conn : getBroadcastConnections()) {
+ conn.setInterestingProperties(new InterestingProperties());
+ }
+ }
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ // check if we have a cached version
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ }
+
+ boolean childrenSkippedDueToReplicatedInput = false;
+
+ // step down to all producer nodes and calculate alternative plans
+ final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
+ final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
+
+ // calculate alternative sub-plans for predecessor
+ final Set<RequestedGlobalProperties> intGlobal1 = this.input1.getInterestingProperties().getGlobalProperties();
+ final Set<RequestedGlobalProperties> intGlobal2 = this.input2.getInterestingProperties().getGlobalProperties();
+
+ // calculate alternative sub-plans for broadcast inputs
+ final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
+ List<String> broadcastConnectionNames = getBroadcastConnectionNames();
+
+ for (int i = 0; i < broadcastConnections.size(); i++ ) {
+ DagConnection broadcastConnection = broadcastConnections.get(i);
+ String broadcastConnectionName = broadcastConnectionNames.get(i);
+ List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
+
+ // wrap the plan candidates in named channels
+ HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
+ for (PlanNode plan: broadcastPlanCandidates) {
+ final NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
+ DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(),
+ ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
+ c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
+ broadcastChannels.add(c);
+ }
+ broadcastPlanChannels.add(broadcastChannels);
+ }
+
+ final GlobalPropertiesPair[] allGlobalPairs;
+ final LocalPropertiesPair[] allLocalPairs;
+ {
+ Set<GlobalPropertiesPair> pairsGlob = new HashSet<GlobalPropertiesPair>();
+ Set<LocalPropertiesPair> pairsLoc = new HashSet<LocalPropertiesPair>();
+ for (OperatorDescriptorDual ods : getProperties()) {
+ pairsGlob.addAll(ods.getPossibleGlobalProperties());
+ pairsLoc.addAll(ods.getPossibleLocalProperties());
+ }
+ allGlobalPairs = pairsGlob.toArray(new GlobalPropertiesPair[pairsGlob.size()]);
+ allLocalPairs = pairsLoc.toArray(new LocalPropertiesPair[pairsLoc.size()]);
+ }
+
+ final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
+
+ final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
+ final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
+
+ final int dop = getParallelism();
+ final int inDop1 = getFirstPredecessorNode().getParallelism();
+ final int inDop2 = getSecondPredecessorNode().getParallelism();
+
+ final boolean dopChange1 = dop != inDop1;
+ final boolean dopChange2 = dop != inDop2;
+
+ final boolean input1breaksPipeline = this.input1.isBreakingPipeline();
+ final boolean input2breaksPipeline = this.input2.isBreakingPipeline();
+
+ // enumerate all pairwise combination of the children's plans together with
+ // all possible operator strategy combination
+
+ // create all candidates
+ for (PlanNode child1 : subPlans1) {
+
+ if (child1.getGlobalProperties().isFullyReplicated()) {
+ // fully replicated input is always locally forwarded if DOP is not changed
+ if (dopChange1) {
+ // can not continue with this child
+ childrenSkippedDueToReplicatedInput = true;
+ continue;
+ } else {
+ this.input1.setShipStrategy(ShipStrategyType.FORWARD);
+ }
+ }
+
+ for (PlanNode child2 : subPlans2) {
+
+ if (child2.getGlobalProperties().isFullyReplicated()) {
+ // fully replicated input is always locally forwarded if DOP is not changed
+ if (dopChange2) {
+ // can not continue with this child
+ childrenSkippedDueToReplicatedInput = true;
+ continue;
+ } else {
+ this.input2.setShipStrategy(ShipStrategyType.FORWARD);
+ }
+ }
+
+ // check that the children go together. that is the case if they build upon the same
+ // candidate at the joined branch plan.
+ if (!areBranchCompatible(child1, child2)) {
+ continue;
+ }
+
+ for (RequestedGlobalProperties igps1: intGlobal1) {
+ // create a candidate channel for the first input. mark it cached, if the connection says so
+ final Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
+ if (this.input1.getShipStrategy() == null) {
+ // free to choose the ship strategy
+ igps1.parameterizeChannel(c1, dopChange1, input1Mode, input1breaksPipeline);
+
+ // if the DOP changed, make sure that we cancel out properties, unless the
+ // ship strategy preserves/establishes them even under changing DOPs
+ if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
+ c1.getGlobalProperties().reset();
+ }
+ }
+ else {
+ // ship strategy fixed by compiler hint
+ ShipStrategyType shipType = this.input1.getShipStrategy();
+ DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipType, input1breaksPipeline);
+ if (this.keys1 != null) {
+ c1.setShipStrategy(shipType, this.keys1.toFieldList(), exMode);
+ }
+ else {
+ c1.setShipStrategy(shipType, exMode);
+ }
+
+ if (dopChange1) {
+ c1.adjustGlobalPropertiesForFullParallelismChange();
+ }
+ }
+
+ for (RequestedGlobalProperties igps2: intGlobal2) {
+ // create a candidate channel for the first input. mark it cached, if the connection says so
+ final Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
+ if (this.input2.getShipStrategy() == null) {
+ // free to choose the ship strategy
+ igps2.parameterizeChannel(c2, dopChange2, input2Mode, input2breaksPipeline);
+
+ // if the DOP changed, make sure that we cancel out properties, unless the
+ // ship strategy preserves/establishes them even under changing DOPs
+ if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
+ c2.getGlobalProperties().reset();
+ }
+ } else {
+ // ship strategy fixed by compiler hint
+ ShipStrategyType shipType = this.input2.getShipStrategy();
+ DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipType, input2breaksPipeline);
+ if (this.keys2 != null) {
+ c2.setShipStrategy(shipType, this.keys2.toFieldList(), exMode);
+ } else {
+ c2.setShipStrategy(shipType, exMode);
+ }
+
+ if (dopChange2) {
+ c2.adjustGlobalPropertiesForFullParallelismChange();
+ }
+ }
+
+ /* ********************************************************************
+ * NOTE: Depending on how we proceed with different partitioning,
+ * we might at some point need a compatibility check between
+ * the pairs of global properties.
+ * *******************************************************************/
+
+ outer:
+ for (GlobalPropertiesPair gpp : allGlobalPairs) {
+ if (gpp.getProperties1().isMetBy(c1.getGlobalProperties()) &&
+ gpp.getProperties2().isMetBy(c2.getGlobalProperties()) )
+ {
+ for (OperatorDescriptorDual desc : getProperties()) {
+ if (desc.areCompatible(gpp.getProperties1(), gpp.getProperties2(),
+ c1.getGlobalProperties(), c2.getGlobalProperties()))
+ {
+ Channel c1Clone = c1.clone();
+ c1Clone.setRequiredGlobalProps(gpp.getProperties1());
+ c2.setRequiredGlobalProps(gpp.getProperties2());
+
+ // we form a valid combination, so create the local candidates
+ // for this
+ addLocalCandidates(c1Clone, c2, broadcastPlanChannels, igps1, igps2,
+ outputPlans, allLocalPairs, estimator);
+ break outer;
+ }
+ }
+ }
+ }
+
+ // break the loop over input2's possible global properties, if the property
+ // is fixed via a hint. All the properties are overridden by the hint anyways,
+ // so we can stop after the first
+ if (this.input2.getShipStrategy() != null) {
+ break;
+ }
+ }
+
+ // break the loop over input1's possible global properties, if the property
+ // is fixed via a hint. All the properties are overridden by the hint anyways,
+ // so we can stop after the first
+ if (this.input1.getShipStrategy() != null) {
+ break;
+ }
+ }
+ }
+ }
+
+ if(outputPlans.isEmpty()) {
+ if(childrenSkippedDueToReplicatedInput) {
+ throw new CompilerException("No plan meeting the requirements could be created @ " + this
+ + ". Most likely reason: Invalid use of replicated input.");
+ } else {
+ throw new CompilerException("No plan meeting the requirements could be created @ " + this
+ + ". Most likely reason: Too restrictive plan hints.");
+ }
+ }
+
+ // cost and prune the plans
+ for (PlanNode node : outputPlans) {
+ estimator.costOperator(node);
+ }
+ prunePlanAlternatives(outputPlans);
+ outputPlans.trimToSize();
+
+ this.cachedPlans = outputPlans;
+ return outputPlans;
+ }
+
+ protected void addLocalCandidates(Channel template1, Channel template2, List<Set<? extends NamedChannel>> broadcastPlanChannels,
+ RequestedGlobalProperties rgps1, RequestedGlobalProperties rgps2,
+ List<PlanNode> target, LocalPropertiesPair[] validLocalCombinations, CostEstimator estimator)
+ {
+ for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
+ final Channel in1 = template1.clone();
+ ilp1.parameterizeChannel(in1);
+
+ for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
+ final Channel in2 = template2.clone();
+ ilp2.parameterizeChannel(in2);
+
+ for (OperatorDescriptorDual dps: getProperties()) {
+ for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
+ if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) &&
+ lpp.getProperties2().isMetBy(in2.getLocalProperties()) )
+ {
+ // valid combination
+ // for non trivial local properties, we need to check that they are co compatible
+ // (such as when some sort order is requested, that both are the same sort order
+ if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(),
+ in1.getLocalProperties(), in2.getLocalProperties()))
+ {
+ // copy, because setting required properties and instantiation may
+ // change the channels and should not affect prior candidates
+ Channel in1Copy = in1.clone();
+ in1Copy.setRequiredLocalProps(lpp.getProperties1());
+
+ Channel in2Copy = in2.clone();
+ in2Copy.setRequiredLocalProps(lpp.getProperties2());
+
+ // all right, co compatible
+ instantiate(dps, in1Copy, in2Copy, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
+ break;
+ }
+ // else cannot use this pair, fall through the loop and try the next one
+ }
+ }
+ }
+ }
+ }
+ }
+
+ protected void instantiate(OperatorDescriptorDual operator, Channel in1, Channel in2,
+ List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
+ RequestedGlobalProperties globPropsReq1, RequestedGlobalProperties globPropsReq2,
+ RequestedLocalProperties locPropsReq1, RequestedLocalProperties locPropsReq2)
+ {
+ final PlanNode inputSource1 = in1.getSource();
+ final PlanNode inputSource2 = in2.getSource();
+
+ for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
+
+ boolean validCombination = true;
+
+ // check whether the broadcast inputs use the same plan candidate at the branching point
+ for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
+ NamedChannel nc = broadcastChannelsCombination.get(i);
+ PlanNode bcSource = nc.getSource();
+
+ if (!(areBranchCompatible(bcSource, inputSource1) || areBranchCompatible(bcSource, inputSource2))) {
+ validCombination = false;
+ break;
+ }
+
+ // check branch compatibility against all other broadcast variables
+ for (int k = 0; k < i; k++) {
+ PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
+
+ if (!areBranchCompatible(bcSource, otherBcSource)) {
+ validCombination = false;
+ break;
+ }
+ }
+ }
+
+ if (!validCombination) {
+ continue;
+ }
+
+ placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
+
+ DualInputPlanNode node = operator.instantiate(in1, in2, this);
+ node.setBroadcastInputs(broadcastChannelsCombination);
+
+ SemanticProperties props = this.getSemanticProperties();
+ GlobalProperties gp1 = in1.getGlobalProperties().clone().filterBySemanticProperties(props, 0);
+ GlobalProperties gp2 = in2.getGlobalProperties().clone().filterBySemanticProperties(props, 1);
+ GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);
+
+ LocalProperties lp1 = in1.getLocalProperties().clone().filterBySemanticProperties(props, 0);
+ LocalProperties lp2 = in2.getLocalProperties().clone().filterBySemanticProperties(props, 1);
+ LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
+
+ node.initProperties(combined, locals);
+ node.updatePropertiesWithUniqueSets(getUniqueFields());
+ target.add(node);
+ }
+ }
+
+ protected void placePipelineBreakersIfNecessary(DriverStrategy strategy, Channel in1, Channel in2) {
+ // before we instantiate, check for deadlocks by tracing back to the open branches and checking
+ // whether either no input, or all of them have a dam
+ if (this.hereJoinedBranches != null && this.hereJoinedBranches.size() > 0) {
+ boolean someDamOnLeftPaths = false;
+ boolean damOnAllLeftPaths = true;
+ boolean someDamOnRightPaths = false;
+ boolean damOnAllRightPaths = true;
+
+ if (strategy.firstDam() == DamBehavior.FULL_DAM || in1.getLocalStrategy().dams() || in1.getTempMode().breaksPipeline()) {
+ someDamOnLeftPaths = true;
+ } else {
+ for (OptimizerNode brancher : this.hereJoinedBranches) {
+ PlanNode candAtBrancher = in1.getSource().getCandidateAtBranchPoint(brancher);
+
+ // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
+ if (candAtBrancher == null) {
+ continue;
+ }
+
+ SourceAndDamReport res = in1.getSource().hasDamOnPathDownTo(candAtBrancher);
+ if (res == NOT_FOUND) {
+ throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
+ } else if (res == FOUND_SOURCE) {
+ damOnAllLeftPaths = false;
+ } else if (res == FOUND_SOURCE_AND_DAM) {
+ someDamOnLeftPaths = true;
+ } else {
+ throw new CompilerException();
+ }
+ }
+ }
+
+ if (strategy.secondDam() == DamBehavior.FULL_DAM || in2.getLocalStrategy().dams() || in2.getTempMode().breaksPipeline()) {
+ someDamOnRightPaths = true;
+ } else {
+ for (OptimizerNode brancher : this.hereJoinedBranches) {
+ PlanNode candAtBrancher = in2.getSource().getCandidateAtBranchPoint(brancher);
+
+ // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
+ if (candAtBrancher == null) {
+ continue;
+ }
+
+ SourceAndDamReport res = in2.getSource().hasDamOnPathDownTo(candAtBrancher);
+ if (res == NOT_FOUND) {
+ throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
+ } else if (res == FOUND_SOURCE) {
+ damOnAllRightPaths = false;
+ } else if (res == FOUND_SOURCE_AND_DAM) {
+ someDamOnRightPaths = true;
+ } else {
+ throw new CompilerException();
+ }
+ }
+ }
+
+ // okay combinations are both all dam or both no dam
+ if ( (damOnAllLeftPaths & damOnAllRightPaths) | (!someDamOnLeftPaths & !someDamOnRightPaths) ) {
+ // good, either both materialize already on the way, or both fully pipeline
+ } else {
+ if (someDamOnLeftPaths & !damOnAllRightPaths) {
+ // right needs a pipeline breaker
+ in2.setTempMode(in2.getTempMode().makePipelineBreaker());
+ }
+
+ if (someDamOnRightPaths & !damOnAllLeftPaths) {
+ // right needs a pipeline breaker
+ in1.setTempMode(in1.getTempMode().makePipelineBreaker());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ // handle the data flow branching for the regular inputs
+ addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
+ addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
+
+ List<UnclosedBranchDescriptor> result1 = getFirstPredecessorNode().getBranchesForParent(getFirstIncomingConnection());
+ List<UnclosedBranchDescriptor> result2 = getSecondPredecessorNode().getBranchesForParent(getSecondIncomingConnection());
+
+ ArrayList<UnclosedBranchDescriptor> inputsMerged = new ArrayList<UnclosedBranchDescriptor>();
+ mergeLists(result1, result2, inputsMerged, true);
+
+ // handle the data flow branching for the broadcast inputs
+ List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(inputsMerged);
+
+ this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return getOperator().getSemanticProperties();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void accept(Visitor<OptimizerNode> visitor) {
+ if (visitor.preVisit(this)) {
+ if (this.input1 == null || this.input2 == null) {
+ throw new CompilerException();
+ }
+
+ getFirstPredecessorNode().accept(visitor);
+ getSecondPredecessorNode().accept(visitor);
+
+ for (DagConnection connection : getBroadcastConnections()) {
+ connection.getSource().accept(visitor);
+ }
+
+ visitor.postVisit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
new file mode 100644
index 0000000..45ecdac
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/UnaryOperatorNode.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+
+public class UnaryOperatorNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> operator;
+
+ private final String name;
+
+
+
+ public UnaryOperatorNode(String name, FieldSet keys, OperatorDescriptorSingle ... operators) {
+ this(name, keys, Arrays.asList(operators));
+ }
+
+ public UnaryOperatorNode(String name, FieldSet keys, List<OperatorDescriptorSingle> operators) {
+ super(keys);
+
+ this.operator = operators;
+ this.name = name;
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.operator;
+ }
+
+ @Override
+ public String getName() {
+ return this.name;
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // we have no estimates by default
+ }
+}
[47/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
index 0dd7e93..ff0e004 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
@@ -47,7 +47,7 @@ import org.apache.flink.api.java.record.operators.ReduceOperator;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
@@ -101,7 +101,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
// ---------- compile plan to nephele job graph to verify that no error is thrown ----------
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
@@ -162,13 +162,13 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
allSinks.add(out3Path);
for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getPactContract()).getFilePath();
+ String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
}
// ---------- compile plan to nephele job graph to verify that no error is thrown ----------
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
@@ -262,7 +262,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
//Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
@@ -350,13 +350,13 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
allSinks.add(out3Path);
for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getPactContract()).getFilePath();
+ String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
}
// ---------- compile plan to nephele job graph to verify that no error is thrown ----------
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
@@ -449,7 +449,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
//Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
@@ -495,7 +495,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
.input2(ma2)
.name("Match 2")
.build();
- mat2.setParameter(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_MERGE);
+ mat2.setParameter(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_MERGE);
FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat2);
@@ -505,7 +505,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
//Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
@@ -555,13 +555,13 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
allSinks.add(out2Path);
for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getPactContract()).getFilePath();
+ String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
}
// ---------- compile plan to nephele job graph to verify that no error is thrown ----------
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
index c17ebe8..3e7da6c 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
@@ -33,7 +33,7 @@ import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.optimizer.dag.TempMode;
import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.operators.DriverStrategy;
@@ -51,7 +51,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
public void testRightSide() {
try {
- Plan plan = getTestPlanRightStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
+ Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
OptimizedPlan oPlan = compileNoStats(plan);
@@ -63,7 +63,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -79,7 +79,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
public void testRightSideCountercheck() {
try {
- Plan plan = getTestPlanRightStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
+ Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
OptimizedPlan oPlan = compileNoStats(plan);
@@ -91,7 +91,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -108,7 +108,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
public void testLeftSide() {
try {
- Plan plan = getTestPlanLeftStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
+ Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
OptimizedPlan oPlan = compileNoStats(plan);
@@ -120,7 +120,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -136,7 +136,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
public void testLeftSideCountercheck() {
try {
- Plan plan = getTestPlanLeftStatic(PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
+ Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
OptimizedPlan oPlan = compileNoStats(plan);
@@ -148,7 +148,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -191,7 +191,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -212,10 +212,10 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
Configuration joinStrategy = new Configuration();
- joinStrategy.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+ joinStrategy.setString(Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
if(strategy != "") {
- joinStrategy.setString(PactCompiler.HINT_LOCAL_STRATEGY, strategy);
+ joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
}
DataSet<Tuple3<Long, Long, Long>> inner = iteration.join(smallInput).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
@@ -243,7 +243,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
Configuration joinStrategy = new Configuration();
- joinStrategy.setString(PactCompiler.HINT_LOCAL_STRATEGY, strategy);
+ joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
DataSet<Tuple3<Long, Long, Long>> inner = smallInput.join(iteration).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
index 5265e3a..565d992 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
@@ -59,9 +59,9 @@ public abstract class CompilerTestBase implements java.io.Serializable {
protected transient DataStatistics dataStats;
- protected transient PactCompiler withStatsCompiler;
+ protected transient Optimizer withStatsCompiler;
- protected transient PactCompiler noStatsCompiler;
+ protected transient Optimizer noStatsCompiler;
private transient int statCounter;
@@ -70,10 +70,10 @@ public abstract class CompilerTestBase implements java.io.Serializable {
@Before
public void setup() {
this.dataStats = new DataStatistics();
- this.withStatsCompiler = new PactCompiler(this.dataStats, new DefaultCostEstimator());
+ this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator());
this.withStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
- this.noStatsCompiler = new PactCompiler(null, new DefaultCostEstimator());
+ this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator());
this.noStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
}
@@ -111,7 +111,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
HashMap<String, ArrayList<PlanNode>> map = new HashMap<String, ArrayList<PlanNode>>();
for (PlanNode n : p.getAllNodes()) {
- Operator<?> c = n.getOriginalOptimizerNode().getPactContract();
+ Operator<?> c = n.getOriginalOptimizerNode().getOperator();
String name = c.getName();
ArrayList<PlanNode> list = map.get(name);
@@ -124,7 +124,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
boolean shouldAdd = true;
for (Iterator<PlanNode> iter = list.iterator(); iter.hasNext();) {
PlanNode in = iter.next();
- if (in.getOriginalOptimizerNode().getPactContract() == c) {
+ if (in.getOriginalOptimizerNode().getOperator() == c) {
// is this the child or is our node the child
if (in instanceof SingleInputPlanNode && n instanceof SingleInputPlanNode) {
SingleInputPlanNode thisNode = (SingleInputPlanNode) n;
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
index 7fa331a..b17e777 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
@@ -30,7 +30,7 @@ import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.PlanNode;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.util.DummyInputFormat;
import org.apache.flink.optimizer.util.DummyMatchStub;
import org.apache.flink.optimizer.util.DummyOutputFormat;
@@ -317,7 +317,7 @@ public class DOPChangeTest extends CompilerTestBase {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
//Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
index b8809d7..aaee975 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.*;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.junit.Test;
@SuppressWarnings("serial")
@@ -41,7 +41,7 @@ public class DisjointDataFlowsTest extends CompilerTestBase {
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
index d97f855..34aa9f8 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
@@ -78,10 +78,10 @@ public class DistinctCompilationTest extends CompilerTestBase implements java.io
assertEquals(new FieldList(0, 1), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -136,13 +136,13 @@ public class DistinctCompilationTest extends CompilerTestBase implements java.io
assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, keyExtractor.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, keyProjector.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -192,10 +192,10 @@ public class DistinctCompilationTest extends CompilerTestBase implements java.io
assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
index 146c085..6dadc19 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.MapOperator;
import org.apache.flink.api.java.record.operators.ReduceOperator;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.util.DummyCrossStub;
import org.apache.flink.optimizer.util.DummyInputFormat;
import org.apache.flink.optimizer.util.DummyOutputFormat;
@@ -74,7 +74,7 @@ public class HardPlansCompilationTest extends CompilerTestBase {
plan.setDefaultParallelism(DEFAULT_PARALLELISM);
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
index dc6fcad..ac4f820 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
@@ -42,7 +42,7 @@ import org.apache.flink.optimizer.plan.Channel;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
import org.apache.flink.optimizer.testfunctions.IdentityMapper;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
@@ -79,7 +79,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(p);
// check that the JobGraphGenerator accepts the plan
- new NepheleJobGraphGenerator().compileJobGraph(p);
+ new JobGraphGenerator().compileJobGraph(p);
}
catch (Exception e) {
e.printStackTrace();
@@ -116,7 +116,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -152,7 +152,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -188,7 +188,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy());
assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -222,7 +222,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy());
}
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -299,7 +299,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
index d6852f5..e65758f 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
@@ -27,7 +27,7 @@ import org.apache.flink.api.java.operators.DeltaIteration;
import org.apache.flink.api.java.operators.IterativeDataSet;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
import org.apache.flink.optimizer.testfunctions.IdentityMapper;
@@ -133,7 +133,7 @@ public class NestedIterationsTest extends CompilerTestBase {
OptimizedPlan op = compileNoStats(p);
// job graph generator should be able to translate this
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -171,7 +171,7 @@ public class NestedIterationsTest extends CompilerTestBase {
OptimizedPlan op = compileNoStats(p);
// job graph generator should be able to translate this
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
index e5983d9..86f01b0 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
@@ -149,7 +149,7 @@ public class PipelineBreakerTest extends CompilerTestBase {
DataSet<Long> initialSource = env.generateSequence(1, 10);
Configuration conf= new Configuration();
- conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST);
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST);
initialSource
.map(new IdentityMapper<Long>())
.cross(initialSource).withParameters(conf)
@@ -171,7 +171,7 @@ public class PipelineBreakerTest extends CompilerTestBase {
DataSet<Long> initialSource = env.generateSequence(1, 10);
Configuration conf= new Configuration();
- conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
initialSource
.map(new IdentityMapper<Long>())
.cross(initialSource).withParameters(conf)
@@ -194,7 +194,7 @@ public class PipelineBreakerTest extends CompilerTestBase {
DataSet<Long> initialSource = env.generateSequence(1, 10);
Configuration conf= new Configuration();
- conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
initialSource
.map(new IdentityMapper<Long>())
.cross(initialSource).withParameters(conf)
@@ -217,7 +217,7 @@ public class PipelineBreakerTest extends CompilerTestBase {
DataSet<Long> initialSource = env.generateSequence(1, 10);
Configuration conf= new Configuration();
- conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
initialSource
.map(new IdentityMapper<Long>())
.cross(initialSource).withParameters(conf)
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
index 23fa311..f6885c5 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.ReduceOperator;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.util.DummyInputFormat;
import org.apache.flink.optimizer.util.DummyOutputFormat;
import org.apache.flink.optimizer.util.IdentityReduce;
@@ -51,7 +51,7 @@ public class ReduceAllTest extends CompilerTestBase {
try {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
} catch(CompilerException ce) {
ce.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
index 8356e94..1fe16bb 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
@@ -63,7 +63,7 @@ public class SemanticPropertiesAPIToPlanTest extends CompilerTestBase {
oPlan.accept(new Visitor<PlanNode>() {
@Override
public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof SingleInputPlanNode && visitable.getPactContract() instanceof ReduceOperatorBase) {
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperatorBase) {
for (Channel input: visitable.getInputs()) {
GlobalProperties gprops = visitable.getGlobalProperties();
LocalProperties lprops = visitable.getLocalProperties();
@@ -78,7 +78,7 @@ public class SemanticPropertiesAPIToPlanTest extends CompilerTestBase {
lprops.getGroupedFields().contains(1));
}
}
- if (visitable instanceof SingleInputPlanNode && visitable.getPactContract() instanceof MapOperatorBase) {
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof MapOperatorBase) {
for (Channel input: visitable.getInputs()) {
GlobalProperties gprops = visitable.getGlobalProperties();
LocalProperties lprops = visitable.getLocalProperties();
@@ -124,7 +124,7 @@ public class SemanticPropertiesAPIToPlanTest extends CompilerTestBase {
oPlan.accept(new Visitor<PlanNode>() {
@Override
public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof DualInputPlanNode && visitable.getPactContract() instanceof JoinOperatorBase) {
+ if (visitable instanceof DualInputPlanNode && visitable.getProgramOperator() instanceof JoinOperatorBase) {
DualInputPlanNode node = ((DualInputPlanNode) visitable);
final Channel inConn1 = node.getInput1();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
index 6647483..92b4fc5 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
@@ -30,7 +30,7 @@ import org.apache.flink.optimizer.plan.Channel;
import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.junit.Test;
@SuppressWarnings("serial")
@@ -80,7 +80,7 @@ public class UnionBetweenDynamicAndStaticPathTest extends CompilerTestBase {
assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -133,7 +133,7 @@ public class UnionBetweenDynamicAndStaticPathTest extends CompilerTestBase {
assertTrue(c.isOnDynamicPath());
}
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
index cb4bce4..5d15ed8 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
@@ -43,7 +43,7 @@ import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.PlanNode;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.util.DummyInputFormat;
import org.apache.flink.optimizer.util.DummyOutputFormat;
import org.apache.flink.optimizer.util.IdentityReduce;
@@ -78,7 +78,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
// Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
@@ -87,7 +87,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
@Override
public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof SingleInputPlanNode && visitable.getPactContract() instanceof ReduceOperator) {
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperator) {
for (Channel inConn : visitable.getInputs()) {
Assert.assertTrue("Reduce should just forward the input if it is already partitioned",
inConn.getShipStrategy() == ShipStrategyType.FORWARD);
@@ -126,7 +126,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
// return the plan
Plan plan = env.createProgramPlan("Test union on new java-api");
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
// Compile plan to verify that no error is thrown
jobGen.compileJobGraph(oPlan);
@@ -139,7 +139,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
/* Test on the union output connections
* It must be under the GroupOperator and the strategy should be forward
*/
- if (visitable instanceof SingleInputPlanNode && visitable.getPactContract() instanceof GroupReduceOperatorBase){
+ if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof GroupReduceOperatorBase){
final Channel inConn = ((SingleInputPlanNode) visitable).getInput();
Assert.assertTrue("Union should just forward the Partitioning",
inConn.getShipStrategy() == ShipStrategyType.FORWARD );
@@ -156,7 +156,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
final Channel inConn = inputs.next();
PlanNode inNode = inConn.getSource();
Assert.assertTrue("Input of Union should be FlatMapOperators",
- inNode.getPactContract() instanceof FlatMapOperatorBase);
+ inNode.getProgramOperator() instanceof FlatMapOperatorBase);
Assert.assertTrue("Shipment strategy under union should partition the data",
inConn.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
index f327259..1e4124c 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
@@ -22,7 +22,7 @@ import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.common.Plan;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.junit.Test;
import static org.junit.Assert.fail;
@@ -44,7 +44,7 @@ public class UnionReplacementTest extends CompilerTestBase {
Plan plan = env.createProgramPlan();
OptimizedPlan oPlan = compileNoStats(plan);
- NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ JobGraphGenerator jobGen = new JobGraphGenerator();
jobGen.compileJobGraph(oPlan);
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
index d10803e..80c0bda 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
@@ -28,7 +28,7 @@ import org.apache.flink.api.java.operators.DeltaIteration;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.junit.Test;
@SuppressWarnings("serial")
@@ -52,7 +52,7 @@ public class WorksetIterationCornerCasesTest extends CompilerTestBase {
WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
assertTrue(wipn.getSolutionSetPlanNode().getOutgoingChannels().isEmpty());
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
jgg.compileJobGraph(op);
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
index f17842e..6e7c0a3 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
@@ -34,7 +34,7 @@ import org.apache.flink.api.java.record.operators.ReduceOperator;
import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.util.DummyInputFormat;
import org.apache.flink.optimizer.util.DummyMatchStub;
import org.apache.flink.optimizer.util.DummyNonPreservingMatchStub;
@@ -106,7 +106,7 @@ public class WorksetIterationsRecordApiCompilerTest extends CompilerTestBase {
assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
(ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
@Test
@@ -150,7 +150,7 @@ public class WorksetIterationsRecordApiCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
@Test
@@ -193,7 +193,7 @@ public class WorksetIterationsRecordApiCompilerTest extends CompilerTestBase {
assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
private Plan getRecordTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
index e876fbb..0273659 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/BinaryCustomPartitioningCompatibilityTest.java
@@ -31,7 +31,7 @@ import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.junit.Test;
@@ -75,7 +75,7 @@ public class BinaryCustomPartitioningCompatibilityTest extends CompilerTestBase
assertEquals(partitioner, partitioner1.getInput().getPartitioner());
assertEquals(partitioner, partitioner2.getInput().getPartitioner());
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -120,7 +120,7 @@ public class BinaryCustomPartitioningCompatibilityTest extends CompilerTestBase
assertEquals(partitioner, partitioner1.getInput().getPartitioner());
assertEquals(partitioner, partitioner2.getInput().getPartitioner());
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
index f6d8d0e..d397ea2 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/custompartition/CustomPartitioningTest.java
@@ -64,17 +64,17 @@ public class CustomPartitioningTest extends CompilerTestBase {
SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getDegreeOfParallelism());
+ assertEquals(parallelism, sink.getParallelism());
assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getDegreeOfParallelism());
+ assertEquals(parallelism, mapper.getParallelism());
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getDegreeOfParallelism());
+ assertEquals(parallelism, partitioner.getParallelism());
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getDegreeOfParallelism());
+ assertEquals(parallelism, balancer.getParallelism());
}
catch (Exception e) {
e.printStackTrace();
@@ -134,17 +134,17 @@ public class CustomPartitioningTest extends CompilerTestBase {
SingleInputPlanNode balancer = (SingleInputPlanNode) partitioner.getInput().getSource();
assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getDegreeOfParallelism());
+ assertEquals(parallelism, sink.getParallelism());
assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getDegreeOfParallelism());
+ assertEquals(parallelism, mapper.getParallelism());
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getDegreeOfParallelism());
+ assertEquals(parallelism, partitioner.getParallelism());
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getDegreeOfParallelism());
+ assertEquals(parallelism, balancer.getParallelism());
}
catch (Exception e) {
e.printStackTrace();
@@ -206,23 +206,23 @@ public class CustomPartitioningTest extends CompilerTestBase {
SingleInputPlanNode balancer = (SingleInputPlanNode) keyExtractor.getInput().getSource();
assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(parallelism, sink.getDegreeOfParallelism());
+ assertEquals(parallelism, sink.getParallelism());
assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
- assertEquals(parallelism, mapper.getDegreeOfParallelism());
+ assertEquals(parallelism, mapper.getParallelism());
assertEquals(ShipStrategyType.FORWARD, keyRemover.getInput().getShipStrategy());
- assertEquals(parallelism, keyRemover.getDegreeOfParallelism());
+ assertEquals(parallelism, keyRemover.getParallelism());
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
assertEquals(part, partitioner.getInput().getPartitioner());
- assertEquals(parallelism, partitioner.getDegreeOfParallelism());
+ assertEquals(parallelism, partitioner.getParallelism());
assertEquals(ShipStrategyType.FORWARD, keyExtractor.getInput().getShipStrategy());
- assertEquals(parallelism, keyExtractor.getDegreeOfParallelism());
+ assertEquals(parallelism, keyExtractor.getParallelism());
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy());
- assertEquals(parallelism, balancer.getDegreeOfParallelism());
+ assertEquals(parallelism, balancer.getParallelism());
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
index 2c5d235..cb4bd78 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeClosedBranchingTest.java
@@ -246,7 +246,7 @@ public class DataExchangeModeClosedBranchingTest extends CompilerTestBase {
private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
for (SinkPlanNode node : collection) {
- String nodeName = node.getOptimizerNode().getPactContract().getName();
+ String nodeName = node.getOptimizerNode().getOperator().getName();
if (nodeName != null && nodeName.equals(name)) {
return node;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
index 6c0e88b..6b2691a 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
@@ -171,7 +171,7 @@ public class DataExchangeModeOpenBranchingTest extends CompilerTestBase {
private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
for (SinkPlanNode node : collection) {
- String nodeName = node.getOptimizerNode().getPactContract().getName();
+ String nodeName = node.getOptimizerNode().getOperator().getName();
if (nodeName != null && nodeName.equals(name)) {
return node;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
index dae3c41..fe33635 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.dag.OptimizerNode;
import org.apache.flink.optimizer.dag.SingleInputNode;
@@ -292,8 +292,8 @@ public class PipelineBreakingTest {
}
private static List<DataSinkNode> convertPlan(Plan p) {
- PactCompiler.GraphCreatingVisitor dagCreator =
- new PactCompiler.GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
+ Optimizer.GraphCreatingVisitor dagCreator =
+ new Optimizer.GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
// create the DAG
p.accept(dagCreator);
@@ -312,8 +312,8 @@ public class PipelineBreakingTest {
rootNode = new SinkJoiner(rootNode, iter.next());
}
}
- rootNode.accept(new PactCompiler.IdAndEstimatesVisitor(null));
- rootNode.accept(new PactCompiler.BranchesVisitor());
+ rootNode.accept(new Optimizer.IdAndEstimatesVisitor(null));
+ rootNode.accept(new Optimizer.BranchesVisitor());
return sinks;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
index 32aeab9..a683968 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
@@ -73,9 +73,9 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(DriverStrategy.ALL_GROUP_REDUCE, reduceNode.getDriverStrategy());
// check DOP
- assertEquals(1, sourceNode.getDegreeOfParallelism());
- assertEquals(1, reduceNode.getDegreeOfParallelism());
- assertEquals(1, sinkNode.getDegreeOfParallelism());
+ assertEquals(1, sourceNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -121,10 +121,10 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(DriverStrategy.ALL_GROUP_REDUCE_COMBINE, combineNode.getDriverStrategy());
// check DOP
- assertEquals(8, sourceNode.getDegreeOfParallelism());
- assertEquals(8, combineNode.getDegreeOfParallelism());
- assertEquals(1, reduceNode.getDegreeOfParallelism());
- assertEquals(1, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, sourceNode.getParallelism());
+ assertEquals(8, combineNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -172,9 +172,9 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -229,10 +229,10 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -285,12 +285,12 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, keyExtractor.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, keyProjector.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -351,13 +351,13 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, keyExtractor.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, keyProjector.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
index 54b24dd..37a8e81 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
@@ -34,7 +34,7 @@ import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
import org.apache.flink.optimizer.plan.SinkPlanNode;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.optimizer.testfunctions.IdentityMapper;
import org.junit.Test;
@@ -53,7 +53,7 @@ public class IterationCompilerTest extends CompilerTestBase {
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -81,7 +81,7 @@ public class IterationCompilerTest extends CompilerTestBase {
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -120,7 +120,7 @@ public class IterationCompilerTest extends CompilerTestBase {
assertTrue(union.getCostWeight() >= 1);
// see that the jobgraph generator can translate this
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
@@ -179,7 +179,7 @@ public class IterationCompilerTest extends CompilerTestBase {
assertTrue(solutionDeltaUnion.isOnDynamicPath());
assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
- new NepheleJobGraphGenerator().compileJobGraph(op);
+ new JobGraphGenerator().compileJobGraph(op);
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
index bbdad4a..0724a9f 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
@@ -72,9 +72,9 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
assertEquals(reduceNode, sinkNode.getInput().getSource());
// check DOP
- assertEquals(1, sourceNode.getDegreeOfParallelism());
- assertEquals(1, reduceNode.getDegreeOfParallelism());
- assertEquals(1, sinkNode.getDegreeOfParallelism());
+ assertEquals(1, sourceNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -122,10 +122,10 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
assertEquals(DriverStrategy.ALL_REDUCE, combineNode.getDriverStrategy());
// check DOP
- assertEquals(8, sourceNode.getDegreeOfParallelism());
- assertEquals(8, combineNode.getDegreeOfParallelism());
- assertEquals(1, reduceNode.getDegreeOfParallelism());
- assertEquals(1, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, sourceNode.getParallelism());
+ assertEquals(8, combineNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -180,10 +180,10 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -244,13 +244,13 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
// check DOP
- assertEquals(6, sourceNode.getDegreeOfParallelism());
- assertEquals(6, keyExtractor.getDegreeOfParallelism());
- assertEquals(6, combineNode.getDegreeOfParallelism());
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getDegreeOfParallelism());
- assertEquals(8, keyProjector.getDegreeOfParallelism());
- assertEquals(8, sinkNode.getDegreeOfParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
}
catch (Exception e) {
System.err.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
index 658cf7a..8720aa7 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/java/WorksetIterationsJavaApiCompilerTest.java
@@ -37,7 +37,7 @@ import org.apache.flink.optimizer.CompilerTestBase;
import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.util.Collector;
import org.junit.Test;
@@ -93,7 +93,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
(ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -137,7 +137,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
@@ -182,7 +182,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
- new NepheleJobGraphGenerator().compileJobGraph(oPlan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
index 21d95ee..e7807c9 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/CoGroupOnConflictingPartitioningsTest.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
import org.apache.flink.configuration.Configuration;
import org.junit.Test;
@@ -42,8 +42,8 @@ public class CoGroupOnConflictingPartitioningsTest extends CompilerTestBase {
DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
Configuration cfg = new Configuration();
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH);
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
input.coGroup(input).where(0).equalTo(0)
.with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
index 4cfa189..9171cc7 100644
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/optimizer/operators/JoinOnConflictingPartitioningsTest.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.configuration.Configuration;
import org.junit.Test;
@@ -41,8 +41,8 @@ public class JoinOnConflictingPartitioningsTest extends CompilerTestBase {
DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
Configuration cfg = new Configuration();
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH);
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
input.join(input).where(0).equalTo(0)
.withParameters(cfg)
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
index 79e04fb..019345f 100644
--- a/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
+++ b/flink-staging/flink-spargel/src/test/java/org/apache/flink/spargel/java/SpargelCompilerTest.java
@@ -68,24 +68,24 @@ public class SpargelCompilerTest extends CompilerTestBase {
// check the sink
SinkPlanNode sink = op.getDataSinks().iterator().next();
assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
// check the iteration
WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
- assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
// check the solution set join and the delta
PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
- assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, ssJoin.getParallelism());
assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
// check the workset set join
DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
- assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, edgeJoin.getParallelism());
assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
assertTrue(edgeJoin.getInput1().getTempMode().isCached());
@@ -143,24 +143,24 @@ public class SpargelCompilerTest extends CompilerTestBase {
// check the sink
SinkPlanNode sink = op.getDataSinks().iterator().next();
assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
- assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, sink.getParallelism());
// check the iteration
WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
- assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, iteration.getParallelism());
// check the solution set join and the delta
PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
- assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, ssJoin.getParallelism());
assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
// check the workset set join
DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
- assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
+ assertEquals(DEFAULT_PARALLELISM, edgeJoin.getParallelism());
assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
assertTrue(edgeJoin.getInput1().getTempMode().isCached());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
index acf20d3..788327a 100644
--- a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
+++ b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
@@ -30,7 +30,7 @@ import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
import org.apache.flink.api.common.operators.GenericDataSourceBase;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.PlanNode;
@@ -57,9 +57,9 @@ public abstract class CompilerTestBase {
protected DataStatistics dataStats;
- protected PactCompiler withStatsCompiler;
+ protected Optimizer withStatsCompiler;
- protected PactCompiler noStatsCompiler;
+ protected Optimizer noStatsCompiler;
private int statCounter;
@@ -68,10 +68,10 @@ public abstract class CompilerTestBase {
@Before
public void setup() {
this.dataStats = new DataStatistics();
- this.withStatsCompiler = new PactCompiler(this.dataStats, new DefaultCostEstimator());
+ this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator());
this.withStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
- this.noStatsCompiler = new PactCompiler(null, new DefaultCostEstimator());
+ this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator());
this.noStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
}
@@ -113,7 +113,7 @@ public abstract class CompilerTestBase {
HashMap<String, ArrayList<PlanNode>> map = new HashMap<String, ArrayList<PlanNode>>();
for (PlanNode n : p.getAllNodes()) {
- Operator<?> c = n.getOriginalOptimizerNode().getPactContract();
+ Operator<?> c = n.getOriginalOptimizerNode().getOperator();
String name = c.getName();
ArrayList<PlanNode> list = map.get(name);
@@ -126,7 +126,7 @@ public abstract class CompilerTestBase {
boolean shouldAdd = true;
for (Iterator<PlanNode> iter = list.iterator(); iter.hasNext();) {
PlanNode in = iter.next();
- if (in.getOriginalOptimizerNode().getPactContract() == c) {
+ if (in.getOriginalOptimizerNode().getOperator() == c) {
// is this the child or is our node the child
if (in instanceof SingleInputPlanNode && n instanceof SingleInputPlanNode) {
SingleInputPlanNode thisNode = (SingleInputPlanNode) n;
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-test-utils/src/main/java/org/apache/flink/test/util/RecordAPITestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/java/org/apache/flink/test/util/RecordAPITestBase.java b/flink-test-utils/src/main/java/org/apache/flink/test/util/RecordAPITestBase.java
index 7cd3ff0..67a4797 100644
--- a/flink-test-utils/src/main/java/org/apache/flink/test/util/RecordAPITestBase.java
+++ b/flink-test-utils/src/main/java/org/apache/flink/test/util/RecordAPITestBase.java
@@ -23,10 +23,10 @@ import akka.actor.ActorRef;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.Plan;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -72,14 +72,14 @@ public abstract class RecordAPITestBase extends AbstractTestBase {
Assert.fail("Error: Cannot obtain Pact plan. Did the thest forget to override either 'getPactPlan()' or 'getJobGraph()' ?");
}
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(p);
if (printPlan) {
System.out.println(new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op));
}
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java b/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
index db1fc4d..44f35e7 100644
--- a/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
+++ b/flink-test-utils/src/main/java/org/apache/flink/test/util/TestEnvironment.java
@@ -24,10 +24,10 @@ import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.ExecutionEnvironmentFactory;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.junit.Assert;
@@ -49,7 +49,7 @@ public class TestEnvironment extends ExecutionEnvironment {
try {
OptimizedPlan op = compileProgram(jobName);
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
JobGraph jobGraph = jgg.compileJobGraph(op);
ActorRef client = this.executor.getJobClient();
@@ -80,7 +80,7 @@ public class TestEnvironment extends ExecutionEnvironment {
private OptimizedPlan compileProgram(String jobName) {
Plan p = createProgramPlan(jobName);
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
return pc.compile(p);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
index 9ab62e1..400ed3a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
@@ -38,9 +38,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.Plan;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.util.StringUtils;
import org.apache.hadoop.fs.FileSystem;
@@ -140,9 +140,9 @@ public abstract class CancellingTestBase {
}
private JobGraph getJobGraph(final Plan plan) throws Exception {
- final PactCompiler pc = new PactCompiler(new DataStatistics());
+ final Optimizer pc = new Optimizer(new DataStatistics());
final OptimizedPlan op = pc.compile(plan);
- final NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ final JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsCoGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsCoGroupTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsCoGroupTest.java
index 1b73c4b..b38b784 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsCoGroupTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsCoGroupTest.java
@@ -27,7 +27,7 @@ import org.apache.flink.optimizer.plan.SinkPlanNode;
import org.apache.flink.optimizer.plan.SourcePlanNode;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
@@ -130,7 +130,7 @@ public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
// check the caches
Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode());
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
jgg.compileJobGraph(optPlan);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsTest.java
index 2c23eaf..dcc9c15 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/ConnectedComponentsTest.java
@@ -40,7 +40,7 @@ import org.apache.flink.optimizer.plan.SinkPlanNode;
import org.apache.flink.optimizer.plan.SourcePlanNode;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
@@ -152,7 +152,7 @@ public class ConnectedComponentsTest extends CompilerTestBase {
Assert.assertTrue(TempMode.PIPELINE_BREAKER == iter.getInitialWorksetInput().getTempMode() ||
LocalStrategy.SORT == iter.getInitialWorksetInput().getLocalStrategy());
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
jgg.compileJobGraph(optPlan);
}
@@ -233,7 +233,7 @@ public class ConnectedComponentsTest extends CompilerTestBase {
Assert.assertTrue(TempMode.PIPELINE_BREAKER == iter.getInitialWorksetInput().getTempMode() ||
LocalStrategy.SORT == iter.getInitialWorksetInput().getLocalStrategy());
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
jgg.compileJobGraph(optPlan);
}
[28/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
deleted file mode 100644
index 565d992..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/CompilerTestBase.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.optimizer.costs.DefaultCostEstimator;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.util.OperatingSystem;
-import org.apache.flink.util.Visitor;
-import org.junit.Before;
-
-/**
- * Base class for Optimizer tests. Offers utility methods to trigger optimization
- * of a program and to fetch the nodes in an optimizer plan that correspond
- * the the node in the program plan.
- */
-public abstract class CompilerTestBase implements java.io.Serializable {
-
- private static final long serialVersionUID = 1L;
-
- protected static final String IN_FILE = OperatingSystem.isWindows() ? "file:/c:/" : "file:///dev/random";
-
- protected static final String OUT_FILE = OperatingSystem.isWindows() ? "file:/c:/" : "file:///dev/null";
-
- protected static final int DEFAULT_PARALLELISM = 8;
-
- private static final String CACHE_KEY = "cachekey";
-
- // ------------------------------------------------------------------------
-
- protected transient DataStatistics dataStats;
-
- protected transient Optimizer withStatsCompiler;
-
- protected transient Optimizer noStatsCompiler;
-
- private transient int statCounter;
-
- // ------------------------------------------------------------------------
-
- @Before
- public void setup() {
- this.dataStats = new DataStatistics();
- this.withStatsCompiler = new Optimizer(this.dataStats, new DefaultCostEstimator());
- this.withStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- this.noStatsCompiler = new Optimizer(null, new DefaultCostEstimator());
- this.noStatsCompiler.setDefaultDegreeOfParallelism(DEFAULT_PARALLELISM);
- }
-
- // ------------------------------------------------------------------------
-
- public OptimizedPlan compileWithStats(Plan p) {
- return this.withStatsCompiler.compile(p);
- }
-
- public OptimizedPlan compileNoStats(Plan p) {
- return this.noStatsCompiler.compile(p);
- }
-
- public void setSourceStatistics(GenericDataSourceBase<?, ?> source, long size, float recordWidth) {
- setSourceStatistics(source, new FileBaseStatistics(Long.MAX_VALUE, size, recordWidth));
- }
-
- public void setSourceStatistics(GenericDataSourceBase<?, ?> source, FileBaseStatistics stats) {
- final String key = CACHE_KEY + this.statCounter++;
- this.dataStats.cacheBaseStatistics(stats, key);
- source.setStatisticsKey(key);
- }
-
- // ------------------------------------------------------------------------
-
- public static OptimizerPlanNodeResolver getOptimizerPlanNodeResolver(OptimizedPlan plan) {
- return new OptimizerPlanNodeResolver(plan);
- }
-
- public static final class OptimizerPlanNodeResolver {
-
- private final Map<String, ArrayList<PlanNode>> map;
-
- OptimizerPlanNodeResolver(OptimizedPlan p) {
- HashMap<String, ArrayList<PlanNode>> map = new HashMap<String, ArrayList<PlanNode>>();
-
- for (PlanNode n : p.getAllNodes()) {
- Operator<?> c = n.getOriginalOptimizerNode().getOperator();
- String name = c.getName();
-
- ArrayList<PlanNode> list = map.get(name);
- if (list == null) {
- list = new ArrayList<PlanNode>(2);
- map.put(name, list);
- }
-
- // check whether this node is a child of a node with the same contract (aka combiner)
- boolean shouldAdd = true;
- for (Iterator<PlanNode> iter = list.iterator(); iter.hasNext();) {
- PlanNode in = iter.next();
- if (in.getOriginalOptimizerNode().getOperator() == c) {
- // is this the child or is our node the child
- if (in instanceof SingleInputPlanNode && n instanceof SingleInputPlanNode) {
- SingleInputPlanNode thisNode = (SingleInputPlanNode) n;
- SingleInputPlanNode otherNode = (SingleInputPlanNode) in;
-
- if (thisNode.getPredecessor() == otherNode) {
- // other node is child, remove it
- iter.remove();
- } else if (otherNode.getPredecessor() == thisNode) {
- shouldAdd = false;
- }
- } else {
- throw new RuntimeException("Unrecodnized case in test.");
- }
- }
- }
-
- if (shouldAdd) {
- list.add(n);
- }
- }
-
- this.map = map;
- }
-
-
- @SuppressWarnings("unchecked")
- public <T extends PlanNode> T getNode(String name) {
- List<PlanNode> nodes = this.map.get(name);
- if (nodes == null || nodes.isEmpty()) {
- throw new RuntimeException("No node found with the given name.");
- } else if (nodes.size() != 1) {
- throw new RuntimeException("Multiple nodes found with the given name.");
- } else {
- return (T) nodes.get(0);
- }
- }
-
- @SuppressWarnings("unchecked")
- public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
- List<PlanNode> nodes = this.map.get(name);
- if (nodes == null || nodes.isEmpty()) {
- throw new RuntimeException("No node found with the given name and stub class.");
- } else {
- PlanNode found = null;
- for (PlanNode node : nodes) {
- if (node.getClass() == stubClass) {
- if (found == null) {
- found = node;
- } else {
- throw new RuntimeException("Multiple nodes found with the given name and stub class.");
- }
- }
- }
- if (found == null) {
- throw new RuntimeException("No node found with the given name and stub class.");
- } else {
- return (T) found;
- }
- }
- }
-
- public List<PlanNode> getNodes(String name) {
- List<PlanNode> nodes = this.map.get(name);
- if (nodes == null || nodes.isEmpty()) {
- throw new RuntimeException("No node found with the given name.");
- } else {
- return new ArrayList<PlanNode>(nodes);
- }
- }
- }
-
- /**
- * Collects all DataSources of a plan to add statistics
- *
- */
- public static class SourceCollectorVisitor implements Visitor<Operator<?>> {
-
- protected final List<GenericDataSourceBase<?, ?>> sources = new ArrayList<GenericDataSourceBase<?, ?>>(4);
-
- @Override
- public boolean preVisit(Operator<?> visitable) {
-
- if(visitable instanceof GenericDataSourceBase) {
- sources.add((GenericDataSourceBase<?, ?>) visitable);
- }
- else if(visitable instanceof BulkIterationBase) {
- ((BulkIterationBase<?>) visitable).getNextPartialSolution().accept(this);
- }
-
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> visitable) {}
-
- public List<GenericDataSourceBase<?, ?>> getSources() {
- return this.sources;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
deleted file mode 100644
index b17e777..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DOPChangeTest.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.optimizer;
-
-import org.junit.Assert;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.JoinOperator;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyMatchStub;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityMap;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.util.Visitor;
-import org.junit.Test;
-
-/**
- * Tests in this class:
- * <ul>
- * <li>Tests that check the correct handling of the properties and strategies in the case where the degree of
- * parallelism between tasks is increased or decreased.
- * </ul>
- */
-@SuppressWarnings({"serial", "deprecation"})
-public class DOPChangeTest extends CompilerTestBase {
-
- /**
- * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
- *
- * Increases DOP between 1st reduce and 2nd map, so the hash partitioning from 1st reduce is not reusable.
- * Expected to re-establish partitioning between reduce and map, via hash, because random is a full network
- * transit as well.
- */
- @Test
- public void checkPropertyHandlingWithIncreasingGlobalParallelism1() {
- final int degOfPar = DEFAULT_PARALLELISM;
-
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
- source.setDegreeOfParallelism(degOfPar);
-
- MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
- map1.setDegreeOfParallelism(degOfPar);
- map1.setInput(source);
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
- reduce1.setDegreeOfParallelism(degOfPar);
- reduce1.setInput(map1);
-
- MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
- map2.setDegreeOfParallelism(degOfPar * 2);
- map2.setInput(reduce1);
-
- ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
- reduce2.setDegreeOfParallelism(degOfPar * 2);
- reduce2.setInput(map2);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setDegreeOfParallelism(degOfPar * 2);
- sink.setInput(reduce2);
-
- Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
- // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
- // mapper respectively reducer
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
- SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
-
- ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
- ShipStrategyType redIn = red2Node.getInput().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, mapIn);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, redIn);
- }
-
- /**
- * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
- *
- * Increases DOP between 2nd map and 2nd reduce, so the hash partitioning from 1st reduce is not reusable.
- * Expected to re-establish partitioning between map and reduce (hash).
- */
- @Test
- public void checkPropertyHandlingWithIncreasingGlobalParallelism2() {
- final int degOfPar = DEFAULT_PARALLELISM;
-
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
- source.setDegreeOfParallelism(degOfPar);
-
- MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
- map1.setDegreeOfParallelism(degOfPar);
- map1.setInput(source);
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
- reduce1.setDegreeOfParallelism(degOfPar);
- reduce1.setInput(map1);
-
- MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
- map2.setDegreeOfParallelism(degOfPar);
- map2.setInput(reduce1);
-
- ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
- reduce2.setDegreeOfParallelism(degOfPar * 2);
- reduce2.setInput(map2);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setDegreeOfParallelism(degOfPar * 2);
- sink.setInput(reduce2);
-
- Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
- // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
- // mapper respectively reducer
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
- SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
-
- ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
- ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, mapIn);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, reduceIn);
- }
-
- /**
- * Simple Job: Map -> Reduce -> Map -> Reduce. All functions preserve all fields (hence all properties).
- *
- * Increases DOP between 1st reduce and 2nd map, such that more tasks are on one instance.
- * Expected to re-establish partitioning between map and reduce via a local hash.
- */
- @Test
- public void checkPropertyHandlingWithIncreasingLocalParallelism() {
- final int degOfPar = 2 * DEFAULT_PARALLELISM;
-
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
- source.setDegreeOfParallelism(degOfPar);
-
- MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
- map1.setDegreeOfParallelism(degOfPar);
- map1.setInput(source);
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
- reduce1.setDegreeOfParallelism(degOfPar);
- reduce1.setInput(map1);
-
- MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
- map2.setDegreeOfParallelism(degOfPar * 2);
- map2.setInput(reduce1);
-
- ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
- reduce2.setDegreeOfParallelism(degOfPar * 2);
- reduce2.setInput(map2);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setDegreeOfParallelism(degOfPar * 2);
- sink.setInput(reduce2);
-
- Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
- // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
- // mapper respectively reducer
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
- SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
-
- ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
- ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
-
- Assert.assertTrue("Invalid ship strategy for an operator.",
- (ShipStrategyType.PARTITION_RANDOM == mapIn && ShipStrategyType.PARTITION_HASH == reduceIn) ||
- (ShipStrategyType.PARTITION_HASH == mapIn && ShipStrategyType.FORWARD == reduceIn));
- }
-
-
-
- @Test
- public void checkPropertyHandlingWithDecreasingDegreeOfParallelism() {
- final int degOfPar = DEFAULT_PARALLELISM;
-
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
- source.setDegreeOfParallelism(degOfPar * 2);
-
- MapOperator map1 = MapOperator.builder(new IdentityMap()).name("Map1").build();
- map1.setDegreeOfParallelism(degOfPar * 2);
- map1.setInput(source);
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 1").build();
- reduce1.setDegreeOfParallelism(degOfPar * 2);
- reduce1.setInput(map1);
-
- MapOperator map2 = MapOperator.builder(new IdentityMap()).name("Map2").build();
- map2.setDegreeOfParallelism(degOfPar);
- map2.setInput(reduce1);
-
- ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce 2").build();
- reduce2.setDegreeOfParallelism(degOfPar);
- reduce2.setInput(map2);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setDegreeOfParallelism(degOfPar);
- sink.setInput(reduce2);
-
- Plan plan = new Plan(sink, "Test Increasing Degree Of Parallelism");
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
- // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
- // mapper respectively reducer
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
- SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
-
- Assert.assertTrue("The no sorting local strategy.",
- LocalStrategy.SORT == red2Node.getInput().getLocalStrategy() ||
- LocalStrategy.SORT == map2Node.getInput().getLocalStrategy());
-
- Assert.assertTrue("The no partitioning ship strategy.",
- ShipStrategyType.PARTITION_HASH == red2Node.getInput().getShipStrategy() ||
- ShipStrategyType.PARTITION_HASH == map2Node.getInput().getShipStrategy());
- }
-
- /**
- * Checks that re-partitioning happens when the inputs of a two-input contract have different DOPs.
- *
- * Test Plan:
- * <pre>
- *
- * (source) -> reduce -\
- * Match -> (sink)
- * (source) -> reduce -/
- *
- * </pre>
- *
- */
- @Test
- public void checkPropertyHandlingWithTwoInputs() {
- // construct the plan
-
- FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
- FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
-
- ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(sourceA)
- .build();
- ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(sourceB)
- .build();
-
- JoinOperator mat = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(redA)
- .input2(redB)
- .build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat);
-
- sourceA.setDegreeOfParallelism(5);
- sourceB.setDegreeOfParallelism(7);
- redA.setDegreeOfParallelism(5);
- redB.setDegreeOfParallelism(7);
-
- mat.setDegreeOfParallelism(5);
-
- sink.setDegreeOfParallelism(5);
-
-
- // return the PACT plan
- Plan plan = new Plan(sink, "Partition on DoP Change");
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- //Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
-
- oPlan.accept(new Visitor<PlanNode>() {
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof DualInputPlanNode) {
- DualInputPlanNode node = (DualInputPlanNode) visitable;
- Channel c1 = node.getInput1();
- Channel c2 = node.getInput2();
-
- Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy());
- Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy());
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
- // DO NOTHING
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
deleted file mode 100644
index aaee975..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DisjointDataFlowsTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class DisjointDataFlowsTest extends CompilerTestBase {
-
- @Test
- public void testDisjointFlows() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // generate two different flows
- env.generateSequence(1, 10).print();
- env.generateSequence(1, 10).print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
deleted file mode 100644
index 34aa9f8..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/DistinctCompilationTest.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.DistinctOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-
-@SuppressWarnings("serial")
-public class DistinctCompilationTest extends CompilerTestBase implements java.io.Serializable {
-
- @Test
- public void testDistinctPlain() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .distinct().name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(0, 1), reduceNode.getKeys(0));
- assertEquals(new FieldList(0, 1), combineNode.getKeys(0));
- assertEquals(new FieldList(0, 1), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testDistinctWithSelectorFunctionKey() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- data
- .distinct(new KeySelector<Tuple2<String,Double>, String>() {
- public String getKey(Tuple2<String, Double> value) { return value.f0; }
- }).name("reducer")
- .print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // get the key extractors and projectors
- SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
- SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, keyExtractor.getInput().getSource());
- assertEquals(keyProjector, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(0), reduceNode.getKeys(0));
- assertEquals(new FieldList(0), combineNode.getKeys(0));
- assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, keyExtractor.getParallelism());
- assertEquals(6, combineNode.getParallelism());
-
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, keyProjector.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-
- @Test
- public void testDistinctWithFieldPositionKeyCombinable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(8);
-
- DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
- .name("source").setParallelism(6);
-
- DistinctOperator<Tuple2<String, Double>> reduced = data
- .distinct(1).name("reducer");
-
- reduced.print().name("sink");
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
-
- // get the original nodes
- SourcePlanNode sourceNode = resolver.getNode("source");
- SingleInputPlanNode reduceNode = resolver.getNode("reducer");
- SinkPlanNode sinkNode = resolver.getNode("sink");
-
- // get the combiner
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
-
- // check wiring
- assertEquals(sourceNode, combineNode.getInput().getSource());
- assertEquals(reduceNode, sinkNode.getInput().getSource());
-
- // check that both reduce and combiner have the same strategy
- assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
- assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
-
- // check the keys
- assertEquals(new FieldList(1), reduceNode.getKeys(0));
- assertEquals(new FieldList(1), combineNode.getKeys(0));
- assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
-
- // check DOP
- assertEquals(6, sourceNode.getParallelism());
- assertEquals(6, combineNode.getParallelism());
- assertEquals(8, reduceNode.getParallelism());
- assertEquals(8, sinkNode.getParallelism());
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
- }
- }
-}
\ No newline at end of file
[29/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
deleted file mode 100644
index c2e81a8..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.util.Visitor;
-
-/**
- * Visitor that computes the interesting properties for each node in the optimizer DAG. On its recursive
- * depth-first descend, it propagates all interesting properties top-down.
- */
-public class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
-
- private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
-
- /**
- * Creates a new visitor that computes the interesting properties for all nodes in the plan.
- * It uses the given cost estimator used to compute the maximal costs for an interesting property.
- *
- * @param estimator
- * The cost estimator to estimate the maximal costs for interesting properties.
- */
- public InterestingPropertyVisitor(CostEstimator estimator) {
- this.estimator = estimator;
- }
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- // The interesting properties must be computed on the descend. In case a node has multiple outputs,
- // that computation must happen during the last descend.
-
- if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
- node.computeUnionOfInterestingPropertiesFromSuccessors();
- node.computeInterestingPropertiesForInputs(this.estimator);
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
deleted file mode 100644
index 58aa3c1..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * This visitor traverses the selected execution plan and finalizes it:
- *
- * <ul>
- * <li>The graph of nodes is double-linked (links from child to parent are inserted).</li>
- * <li>If unions join static and dynamic paths, the cache is marked as a memory consumer.</li>
- * <li>Relative memory fractions are assigned to all nodes.</li>
- * <li>All nodes are collected into a set.</li>
- * </ul>
- */
-public class PlanFinalizer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
-
- private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
-
- private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
-
- private final Deque<IterationPlanNode> stackOfIterationNodes;
-
- private int memoryConsumerWeights; // a counter of all memory consumers
-
- /**
- * Creates a new plan finalizer.
- */
- public PlanFinalizer() {
- this.allNodes = new HashSet<PlanNode>();
- this.sources = new ArrayList<SourcePlanNode>();
- this.sinks = new ArrayList<SinkPlanNode>();
- this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
- }
-
- public OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
- this.memoryConsumerWeights = 0;
-
- // traverse the graph
- for (SinkPlanNode node : sinks) {
- node.accept(this);
- }
-
- // assign the memory to each node
- if (this.memoryConsumerWeights > 0) {
- for (PlanNode node : this.allNodes) {
- // assign memory to the driver strategy of the node
- final int consumerWeight = node.getMemoryConsumerWeight();
- if (consumerWeight > 0) {
- final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
- node.setRelativeMemoryPerSubtask(relativeMem);
- if (Optimizer.LOG.isDebugEnabled()) {
- Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
- node.getProgramOperator().getName() + ".");
- }
- }
-
- // assign memory to the local and global strategies of the channels
- for (Channel c : node.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- final double relativeMem = 1.0 / this.memoryConsumerWeights;
- c.setRelativeMemoryLocalStrategy(relativeMem);
- if (Optimizer.LOG.isDebugEnabled()) {
- Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
- "instance of " + c + ".");
- }
- }
- if (c.getTempMode() != TempMode.NONE) {
- final double relativeMem = 1.0/ this.memoryConsumerWeights;
- c.setRelativeTempMemory(relativeMem);
- if (Optimizer.LOG.isDebugEnabled()) {
- Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
- "table for " + c + ".");
- }
- }
- }
- }
- }
- return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
- }
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- // if we come here again, prevent a further descend
- if (!this.allNodes.add(visitable)) {
- return false;
- }
-
- if (visitable instanceof SinkPlanNode) {
- this.sinks.add((SinkPlanNode) visitable);
- }
- else if (visitable instanceof SourcePlanNode) {
- this.sources.add((SourcePlanNode) visitable);
- }
- else if (visitable instanceof BinaryUnionPlanNode) {
- BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- if (unionNode.unionsStaticAndDynamicPath()) {
- unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
- }
- }
- else if (visitable instanceof BulkPartialSolutionPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
- }
- else if (visitable instanceof WorksetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
- else if (visitable instanceof SolutionSetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
-
- // double-connect the connections. previously, only parents knew their children, because
- // one child candidate could have been referenced by multiple parents.
- for (Channel conn : visitable.getInputs()) {
- conn.setTarget(visitable);
- conn.getSource().addOutgoingChannel(conn);
- }
-
- for (Channel c : visitable.getBroadcastInputs()) {
- c.setTarget(visitable);
- c.getSource().addOutgoingChannel(c);
- }
-
- // count the memory consumption
- this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
- for (Channel c : visitable.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
- for (Channel c : visitable.getBroadcastInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
-
- // pass the visitor to the iteraton's step function
- if (visitable instanceof IterationPlanNode) {
- // push the iteration node onto the stack
- final IterationPlanNode iterNode = (IterationPlanNode) visitable;
- this.stackOfIterationNodes.addLast(iterNode);
-
- // recurse
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
-
- // pop the iteration node from the stack
- this.stackOfIterationNodes.removeLast();
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
deleted file mode 100644
index c0dc4dd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.util.Visitor;
-
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * A traversal that goes over the program data flow of an iteration and makes the nodes
- * that depend on the partial solution (the data set recomputed in each iteration) as "dynamic"
- * and the other nodes as "static".
- */
-public class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
-
- private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
-
- private final int costWeight;
-
- public StaticDynamicPathIdentifier(int costWeight) {
- this.costWeight = costWeight;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return this.seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- visitable.identifyDynamicPath(this.costWeight);
-
- // check that there is no nested iteration on the dynamic path
- if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
- throw new CompilerException("Nested iterations are currently not supported.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
deleted file mode 100644
index d359490..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.traversals;
-
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.util.Visitor;
-
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * A traversal that checks if the Workset of a delta iteration is used in the data flow
- * of its step function.
- */
-public class StepFunctionValidator implements Visitor<Operator<?>> {
-
- private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
-
- private boolean foundWorkset;
-
- @Override
- public boolean preVisit(Operator<?> visitable) {
- if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
- foundWorkset = true;
- }
-
- return (!foundWorkset) && seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(Operator<?> visitable) {}
-
- public boolean hasFoundWorkset() {
- return foundWorkset;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
deleted file mode 100644
index cd8766c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains the various traversals over the program plan and the
- * optimizer DAG (directed acyclic graph) that are made in the course of
- * the optimization.
- *
- * The traversals are mostly implemented as a {@link org.apache.flink.util.Visitor} that
- * traversed the program flow.
- */
-package org.apache.flink.optimizer.traversals;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
deleted file mode 100644
index 5110849..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.util.List;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.functions.util.NoOpFunction;
-import org.apache.flink.api.common.operators.BinaryOperatorInformation;
-import org.apache.flink.api.common.operators.DualInputOperator;
-import org.apache.flink.api.common.operators.RecordOperator;
-import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.types.Key;
-
-
-public class NoOpBinaryUdfOp<OUT> extends DualInputOperator<OUT, OUT, OUT, NoOpFunction> implements RecordOperator {
-
- public NoOpBinaryUdfOp(TypeInformation<OUT> type) {
- super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), new BinaryOperatorInformation<OUT, OUT, OUT>(type, type, type), "NoContract");
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Class<? extends Key<?>>[] getKeyClasses() {
- return (Class<? extends Key<?>>[]) new Class[0];
- }
-
- @Override
- protected List<OUT> executeOnCollections(List<OUT> inputData1, List<OUT> inputData2, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {
- throw new UnsupportedOperationException();
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
deleted file mode 100644
index cc4a4d6..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.util;
-
-import java.util.List;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.functions.util.NoOpFunction;
-import org.apache.flink.api.common.operators.RecordOperator;
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.api.common.operators.UnaryOperatorInformation;
-import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.types.Key;
-
-
-public class NoOpUnaryUdfOp<OUT> extends SingleInputOperator<OUT, OUT, NoOpFunction> implements RecordOperator {
-
- @SuppressWarnings("rawtypes")
- public static final NoOpUnaryUdfOp INSTANCE = new NoOpUnaryUdfOp();
-
- private NoOpUnaryUdfOp() {
- // pass null here because we override getOutputType to return type
- // of input operator
- super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), null, "");
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Class<? extends Key<?>>[] getKeyClasses() {
- return (Class<? extends Key<?>>[]) new Class[0];
- }
-
- @Override
- public UnaryOperatorInformation<OUT, OUT> getOperatorInfo() {
- TypeInformation<OUT> previousOut = input.getOperatorInfo().getOutputType();
- return new UnaryOperatorInformation<OUT, OUT>(previousOut, previousOut);
- }
-
- @Override
- protected List<OUT> executeOnCollections(List<OUT> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {
- return inputData;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/Utils.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/Utils.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/util/Utils.java
deleted file mode 100644
index d8f33a2..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/util/Utils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.util;
-
-import java.util.Arrays;
-
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-
-
-/**
- *
- */
-public class Utils
-{
- public static final FieldList createOrderedFromSet(FieldSet set) {
- if (set instanceof FieldList) {
- return (FieldList) set;
- } else {
- final int[] cols = set.toArray();
- Arrays.sort(cols);
- return new FieldList(cols);
- }
- }
-
- public static final Ordering createOrdering(FieldList fields, boolean[] directions) {
- final Ordering o = new Ordering();
- for (int i = 0; i < fields.size(); i++) {
- o.appendOrdering(fields.get(i), null, directions == null || directions[i] ? Order.ASCENDING : Order.DESCENDING);
- }
- return o;
- }
-
- public static final Ordering createOrdering(FieldList fields) {
- final Ordering o = new Ordering();
- for (int i = 0; i < fields.size(); i++) {
- o.appendOrdering(fields.get(i), null, Order.ANY);
- }
- return o;
- }
-
- public static boolean[] getDirections(Ordering o, int numFields) {
- final boolean[] dirs = o.getFieldSortDirections();
- if (dirs.length == numFields) {
- return dirs;
- } else if (dirs.length > numFields) {
- final boolean[] subSet = new boolean[numFields];
- System.arraycopy(dirs, 0, subSet, 0, numFields);
- return subSet;
- } else {
- throw new CompilerException();
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * No instantiation.
- */
- private Utils() {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
deleted file mode 100644
index 1e4bafb..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.record.operators.CrossOperator;
-import org.apache.flink.api.java.record.operators.CrossWithLargeOperator;
-import org.apache.flink.api.java.record.operators.CrossWithSmallOperator;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.util.DummyCrossStub;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-/**
-* Tests that validate optimizer choices when using operators that are requesting certain specific execution
-* strategies.
-*/
-@SuppressWarnings({"serial", "deprecation"})
-public class AdditionalOperatorsTest extends CompilerTestBase {
-
- @Test
- public void testCrossWithSmall() {
- // construct the plan
- FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
- FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
-
- CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
- .input1(source1).input2(source2)
- .name("Cross").build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
-
- Plan plan = new Plan(sink);
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
-
- try {
- OptimizedPlan oPlan = compileNoStats(plan);
- OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
-
- DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
- Channel in1 = crossPlanNode.getInput1();
- Channel in2 = crossPlanNode.getInput2();
-
- assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
- assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- }
- }
-
- @Test
- public void testCrossWithLarge() {
- // construct the plan
- FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
- FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
-
- CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
- .input1(source1).input2(source2)
- .name("Cross").build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
-
- Plan plan = new Plan(sink);
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
-
- try {
- OptimizedPlan oPlan = compileNoStats(plan);
- OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
-
- DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
- Channel in1 = crossPlanNode.getInput1();
- Channel in2 = crossPlanNode.getInput2();
-
- assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
deleted file mode 100644
index 916aa27..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
+++ /dev/null
@@ -1,1039 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.common.functions.CoGroupFunction;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
-import org.apache.flink.util.Collector;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichJoinFunction;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
-import org.apache.flink.api.java.record.operators.BulkIteration;
-import org.apache.flink.api.java.record.operators.CoGroupOperator;
-import org.apache.flink.api.java.record.operators.CrossOperator;
-import org.apache.flink.api.java.record.operators.DeltaIteration;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.JoinOperator;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
-import org.apache.flink.optimizer.util.DummyCoGroupStub;
-import org.apache.flink.optimizer.util.DummyCrossStub;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyMatchStub;
-import org.apache.flink.optimizer.util.DummyNonPreservingMatchStub;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityMap;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-
-@SuppressWarnings({"serial", "deprecation"})
-public class BranchingPlansCompilerTest extends CompilerTestBase {
-
-
- @Test
- public void testCostComputationWithMultipleDataSinks() {
- final int SINKS = 5;
-
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSet<Long> source = env.generateSequence(1, 10000);
-
- DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
- DataSet<Long> mappedC = source.map(new IdentityMapper<Long>());
-
- for (int sink = 0; sink < SINKS; sink++) {
- mappedA.output(new DiscardingOutputFormat<Long>());
- mappedC.output(new DiscardingOutputFormat<Long>());
- }
-
- Plan plan = env.createProgramPlan("Plans With Multiple Data Sinks");
- OptimizedPlan oPlan = compileNoStats(plan);
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
-
- /**
- *
- * <pre>
- * (SRC A)
- * |
- * (MAP A)
- * / \
- * (MAP B) (MAP C)
- * / / \
- * (SINK A) (SINK B) (SINK C)
- * </pre>
- */
- @SuppressWarnings("unchecked")
- @Test
- public void testBranchingWithMultipleDataSinks2() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSet<Long> source = env.generateSequence(1, 10000);
-
- DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
- DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
- DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());
-
- mappedB.output(new DiscardingOutputFormat<Long>());
- mappedC.output(new DiscardingOutputFormat<Long>());
- mappedC.output(new DiscardingOutputFormat<Long>());
-
- Plan plan = env.createProgramPlan();
- Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // ---------- check the optimizer plan ----------
-
- // number of sinks
- assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());
-
- // remove matching sinks to check relation
- for (SinkPlanNode sink : oPlan.getDataSinks()) {
- assertTrue(sinks.remove(sink.getProgramOperator()));
- }
- assertTrue(sinks.isEmpty());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
-
- /**
- * <pre>
- * SINK
- * |
- * COGROUP
- * +---/ \----+
- * / \
- * / MATCH10
- * / | \
- * / | MATCH9
- * MATCH5 | | \
- * | \ | | MATCH8
- * | MATCH4 | | | \
- * | | \ | | | MATCH7
- * | | MATCH3 | | | | \
- * | | | \ | | | | MATCH6
- * | | | MATCH2 | | | | | |
- * | | | | \ +--+--+--+--+--+
- * | | | | MATCH1 MAP
- * \ | | | | | /-----------/
- * (DATA SOURCE ONE)
- * </pre>
- */
- @Test
- public void testBranchingSourceMultipleTimes() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000)
- .map(new Duplicator<Long>());
-
- DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> mapped = source.map(
- new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
- return null;
- }
- });
-
- DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
-
- joined5.coGroup(joined10)
- .where(1).equalTo(1)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
-
- .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
-
- Plan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileNoStats(plan);
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- /**
- *
- * <pre>
-
- * (SINK A)
- * | (SINK B) (SINK C)
- * CROSS / /
- * / \ | +------+
- * / \ | /
- * REDUCE MATCH2
- * | +---/ \
- * \ / |
- * MAP |
- * | |
- * COGROUP MATCH1
- * / \ / \
- * (SRC A) (SRC B) (SRC C)
- * </pre>
- */
- @Test
- public void testBranchingWithMultipleDataSinks() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
- .map(new Duplicator<Long>());
-
- DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
- .map(new Duplicator<Long>());
-
- DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
- .map(new Duplicator<Long>());
-
- DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
- .where(0).equalTo(1)
- .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
- @Override
- public void coGroup(Iterable<Tuple2<Long, Long>> first,
- Iterable<Tuple2<Long, Long>> second,
- Collector<Tuple2<Long, Long>> out) {
- }
- })
- .map(new IdentityMapper<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
- .where(0).equalTo(1)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
- .where(1).equalTo(1)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> reduced = mapped
- .groupBy(1)
- .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());
-
- reduced.cross(joined2)
- .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
-
- joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
- joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
-
- Plan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileNoStats(plan);
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testBranchEachContractType() {
- try {
- // construct the plan
- FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), "file:///test/file1", "Source A");
- FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
- FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
-
- MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
-
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(map1)
- .name("Reduce 1")
- .build();
-
- JoinOperator match1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(sourceB, sourceB, sourceC)
- .input2(sourceC)
- .name("Match 1")
- .build();
- ;
- CoGroupOperator cogroup1 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(sourceA)
- .input2(sourceB)
- .name("CoGroup 1")
- .build();
-
- CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub())
- .input1(reduce1)
- .input2(cogroup1)
- .name("Cross 1")
- .build();
-
-
- CoGroupOperator cogroup2 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(cross1)
- .input2(cross1)
- .name("CoGroup 2")
- .build();
-
- CoGroupOperator cogroup3 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(map1)
- .input2(match1)
- .name("CoGroup 3")
- .build();
-
-
- MapOperator map2 = MapOperator.builder(new IdentityMap()).input(cogroup3).name("Map 2").build();
-
- CoGroupOperator cogroup4 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(map2)
- .input2(match1)
- .name("CoGroup 4")
- .build();
-
- CoGroupOperator cogroup5 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(cogroup2)
- .input2(cogroup1)
- .name("CoGroup 5")
- .build();
-
- CoGroupOperator cogroup6 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(reduce1)
- .input2(cogroup4)
- .name("CoGroup 6")
- .build();
-
- CoGroupOperator cogroup7 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
- .input1(cogroup5)
- .input2(cogroup6)
- .name("CoGroup 7")
- .build();
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cogroup7);
- sink.addInput(sourceA);
- sink.addInput(cogroup3);
- sink.addInput(cogroup4);
- sink.addInput(cogroup1);
-
- // return the PACT plan
- Plan plan = new Plan(sink, "Branching of each contract type");
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- //Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
-
- @Test
- public void testBranchingUnion() {
- try {
- // construct the plan
- FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE);
- FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE);
-
- JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(source1)
- .input2(source2)
- .name("Match 1")
- .build();
-
- MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
-
- ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(ma1)
- .name("Reduce 1")
- .build();
-
- ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(mat1)
- .name("Reduce 2")
- .build();
-
- MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
-
- MapOperator ma3 = MapOperator.builder(new IdentityMap()).input(ma2).name("Map 3").build();
-
- @SuppressWarnings("unchecked")
- JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
- .input1(r1, r2, ma2, ma3)
- .input2(ma2)
- .name("Match 2")
- .build();
- mat2.setParameter(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_MERGE);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat2);
-
-
- // return the PACT plan
- Plan plan = new Plan(sink, "Branching Union");
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- //Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- *
- * <pre>
- * (SRC A)
- * / \
- * (SINK A) (SINK B)
- * </pre>
- */
- @Test
- public void testBranchingWithMultipleDataSinksSmall() {
- try {
- // construct the plan
- final String out1Path = "file:///test/1";
- final String out2Path = "file:///test/2";
-
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
-
- FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
- FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceA);
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sinkA);
- sinks.add(sinkB);
-
- // return the PACT plan
- Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // ---------- check the optimizer plan ----------
-
- // number of sinks
- Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
-
- // sinks contain all sink paths
- Set<String> allSinks = new HashSet<String>();
- allSinks.add(out1Path);
- allSinks.add(out2Path);
-
- for (SinkPlanNode n : oPlan.getDataSinks()) {
- String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
- Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
- }
-
- // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- *
- * <pre>
- * (SINK 3) (SINK 1) (SINK 2) (SINK 4)
- * \ / \ /
- * (SRC A) (SRC B)
- * </pre>
- *
- * NOTE: this case is currently not caught by the compiler. we should enable the test once it is caught.
- */
- @Test
- public void testBranchingDisjointPlan() {
- // construct the plan
- final String out1Path = "file:///test/1";
- final String out2Path = "file:///test/2";
- final String out3Path = "file:///test/3";
- final String out4Path = "file:///test/4";
-
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
- FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
-
- FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA, "1");
- FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB, "2");
- FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, out3Path, sourceA, "3");
- FileDataSink sink4 = new FileDataSink(DummyOutputFormat.class, out4Path, sourceB, "4");
-
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink1);
- sinks.add(sink2);
- sinks.add(sink3);
- sinks.add(sink4);
-
- // return the PACT plan
- Plan plan = new Plan(sinks, "Disjoint plan with multiple data sinks and branches");
- compileNoStats(plan);
- }
-
- @Test
- public void testBranchAfterIteration() {
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
-
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(sourceA);
- iteration.setMaximumNumberOfIterations(10);
-
- MapOperator mapper = MapOperator.builder(IdentityMap.class).name("Mapper").input(iteration.getPartialSolution()).build();
- iteration.setNextPartialSolution(mapper);
-
- FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 1");
-
- MapOperator postMap = MapOperator.builder(IdentityMap.class).name("Post Iteration Mapper")
- .input(iteration).build();
-
- FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink 2");
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink1);
- sinks.add(sink2);
-
- Plan plan = new Plan(sinks);
-
- try {
- compileNoStats(plan);
- }
- catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- @Test
- public void testBranchBeforeIteration() {
- FileDataSource source1 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
- FileDataSource source2 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
-
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(source2);
- iteration.setMaximumNumberOfIterations(10);
-
- MapOperator inMap = MapOperator.builder(new IdentityMap())
- .input(source1)
- .name("In Iteration Map")
- .setBroadcastVariable("BC", iteration.getPartialSolution())
- .build();
-
- iteration.setNextPartialSolution(inMap);
-
- MapOperator postMap = MapOperator.builder(new IdentityMap())
- .input(source1)
- .name("Post Iteration Map")
- .setBroadcastVariable("BC", iteration)
- .build();
-
- FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink");
-
- Plan plan = new Plan(sink);
-
- try {
- compileNoStats(plan);
- }
- catch (Exception e) {
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- * Test to ensure that sourceA is inside as well as outside of the iteration the same
- * node.
- *
- * <pre>
- * (SRC A) (SRC B)
- * / \ / \
- * (SINK 1) (ITERATION) | (SINK 2)
- * / \ /
- * (SINK 3) (CROSS => NEXT PARTIAL SOLUTION)
- * </pre>
- */
- @Test
- public void testClosure() {
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
- FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
-
- FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
- FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceB, "Sink 2");
-
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(sourceA);
- iteration.setMaximumNumberOfIterations(10);
-
- CrossOperator stepFunction = CrossOperator.builder(DummyCrossStub.class).name("StepFunction").
- input1(iteration.getPartialSolution()).
- input2(sourceB).
- build();
-
- iteration.setNextPartialSolution(stepFunction);
-
- FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink1);
- sinks.add(sink2);
- sinks.add(sink3);
-
- Plan plan = new Plan(sinks);
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- * <pre>
- * (SRC A) (SRC B) (SRC C)
- * / \ / / \
- * (SINK 1) (DELTA ITERATION) | (SINK 2)
- * / | \ /
- * (SINK 3) | (CROSS => NEXT WORKSET)
- * | |
- * (JOIN => SOLUTION SET DELTA)
- * </pre>
- */
- @Test
- public void testClosureDeltaIteration() {
- FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
- FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
- FileDataSource sourceC = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 3");
-
- FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
- FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceC, "Sink 2");
-
- DeltaIteration iteration = new DeltaIteration(0, "Loop");
- iteration.setInitialSolutionSet(sourceA);
- iteration.setInitialWorkset(sourceB);
- iteration.setMaximumNumberOfIterations(10);
-
- CrossOperator nextWorkset = CrossOperator.builder(DummyCrossStub.class).name("Next workset").
- input1(iteration.getWorkset()).
- input2(sourceC).
- build();
-
- JoinOperator solutionSetDelta = JoinOperator.builder(DummyMatchStub.class, LongValue.class,0,0).
- name("Next solution set.").
- input1(nextWorkset).
- input2(iteration.getSolutionSet()).
- build();
-
- iteration.setNextWorkset(nextWorkset);
- iteration.setSolutionSetDelta(solutionSetDelta);
-
- FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");
-
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink1);
- sinks.add(sink2);
- sinks.add(sink3);
-
- Plan plan = new Plan(sinks);
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- * <pre>
- * +----Iteration-------+
- * | |
- * /---------< >---------join-----< >---sink
- * / (Solution)| / |
- * / | / |
- * /--map-------< >----\ / /--|
- * / (Workset)| \ / / |
- * src-map | join------/ |
- * \ | / |
- * \ +-----/--------------+
- * \ /
- * \--reduce-------/
- * </pre>
- */
- @Test
- public void testDeltaIterationWithStaticInput() {
- FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");
-
- MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
- input(source).
- name("Identity mapped source").
- build();
-
- ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
- input(source).
- name("Identity reduce source").
- build();
-
- DeltaIteration iteration = new DeltaIteration(0,"Loop");
- iteration.setMaximumNumberOfIterations(10);
- iteration.setInitialSolutionSet(source);
- iteration.setInitialWorkset(mappedSource);
-
- JoinOperator nextWorkset = JoinOperator.builder(DummyNonPreservingMatchStub.class, IntValue.class, 0,0).
- input1(iteration.getWorkset()).
- input2(reducedSource).
- name("Next work set").
- build();
-
- JoinOperator solutionSetDelta = JoinOperator.builder(DummyNonPreservingMatchStub.class, IntValue.class, 0,
- 0).
- input1(iteration.getSolutionSet()).
- input2(nextWorkset).
- name("Solution set delta").
- build();
-
- iteration.setNextWorkset(nextWorkset);
- iteration.setSolutionSetDelta(solutionSetDelta);
-
- FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Iteration sink");
- List<FileDataSink> sinks = new ArrayList<FileDataSink>();
- sinks.add(sink);
-
- Plan plan = new Plan(sinks);
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- /**
- * <pre>
- * +---------Iteration-------+
- * | |
- * /--map--< >----\ |
- * / | \ /-------< >---sink
- * src-map | join------/ |
- * \ | / |
- * \ +-----/-------------------+
- * \ /
- * \--reduce--/
- * </pre>
- */
- @Test
- public void testIterationWithStaticInput() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<Long> source = env.generateSequence(1, 1000000);
-
- DataSet<Long> mapped = source.map(new IdentityMapper<Long>());
-
- DataSet<Long> reduced = source.groupBy(new IdentityKeyExtractor<Long>()).reduce(new SelectOneReducer<Long>());
-
- IterativeDataSet<Long> iteration = mapped.iterate(10);
- iteration.closeWith(
- iteration.join(reduced)
- .where(new IdentityKeyExtractor<Long>())
- .equalTo(new IdentityKeyExtractor<Long>())
- .with(new DummyFlatJoinFunction<Long>()))
- .output(new DiscardingOutputFormat<Long>());
-
- compileNoStats(env.createProgramPlan());
- }
- catch(Exception e){
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testBranchingBroadcastVariable() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
- DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
- DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
-
- DataSet<String> result1 = input1
- .map(new IdentityMapper<String>())
- .reduceGroup(new Top1GroupReducer<String>())
- .withBroadcastSet(input3, "bc");
-
- DataSet<String> result2 = input2
- .map(new IdentityMapper<String>())
- .reduceGroup(new Top1GroupReducer<String>())
- .withBroadcastSet(input3, "bc");
-
- result1.join(result2)
- .where(new IdentityKeyExtractor<String>())
- .equalTo(new IdentityKeyExtractor<String>())
- .with(new RichJoinFunction<String, String, String>() {
- @Override
- public String join(String first, String second) {
- return null;
- }
- })
- .withBroadcastSet(input3, "bc1")
- .withBroadcastSet(input1, "bc2")
- .withBroadcastSet(result1, "bc3")
- .print();
-
- Plan plan = env.createProgramPlan();
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- @Test
- public void testBCVariableClosure() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
-
- DataSet<String> reduced = input
- .map(new IdentityMapper<String>())
- .reduceGroup(new Top1GroupReducer<String>());
-
-
- DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
-
-
- IterativeDataSet<String> iteration = initialSolution.iterate(100);
-
- iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
- .print();
-
- Plan plan = env.createProgramPlan();
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- @Test
- public void testMultipleIterations() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
-
- DataSet<String> reduced = input
- .map(new IdentityMapper<String>())
- .reduceGroup(new Top1GroupReducer<String>());
-
- IterativeDataSet<String> iteration1 = input.iterate(100);
- IterativeDataSet<String> iteration2 = input.iterate(20);
- IterativeDataSet<String> iteration3 = input.iterate(17);
-
- iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")).print();
- iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")).print();
- iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")).print();
-
- Plan plan = env.createProgramPlan();
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- @Test
- public void testMultipleIterationsWithClosueBCVars() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
-
- IterativeDataSet<String> iteration1 = input.iterate(100);
- IterativeDataSet<String> iteration2 = input.iterate(20);
- IterativeDataSet<String> iteration3 = input.iterate(17);
-
-
- iteration1.closeWith(iteration1.map(new IdentityMapper<String>())).print();
- iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())).print();
- iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())).print();
-
- Plan plan = env.createProgramPlan();
-
- try{
- compileNoStats(plan);
- }catch(Exception e){
- e.printStackTrace();
- Assert.fail(e.getMessage());
- }
- }
-
- @Test
- public void testBranchesOnlyInBCVariables1() {
- try{
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<Long> input = env.generateSequence(1, 10);
- DataSet<Long> bc_input = env.generateSequence(1, 10);
-
- input
- .map(new IdentityMapper<Long>()).withBroadcastSet(bc_input, "name1")
- .map(new IdentityMapper<Long>()).withBroadcastSet(bc_input, "name2")
- .print();
-
- Plan plan = env.createProgramPlan();
- compileNoStats(plan);
- }
- catch(Exception e){
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testBranchesOnlyInBCVariables2() {
- try{
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(100);
-
- DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 10).map(new Duplicator<Long>()).name("proper input");
-
- DataSet<Long> bc_input1 = env.generateSequence(1, 10).name("BC input 1");
- DataSet<Long> bc_input2 = env.generateSequence(1, 10).name("BC input 1");
-
- DataSet<Tuple2<Long, Long>> joinInput1 =
- input.map(new IdentityMapper<Tuple2<Long,Long>>())
- .withBroadcastSet(bc_input1.map(new IdentityMapper<Long>()), "bc1")
- .withBroadcastSet(bc_input2, "bc2");
-
- DataSet<Tuple2<Long, Long>> joinInput2 =
- input.map(new IdentityMapper<Tuple2<Long,Long>>())
- .withBroadcastSet(bc_input1, "bc1")
- .withBroadcastSet(bc_input2, "bc2");
-
- DataSet<Tuple2<Long, Long>> joinResult = joinInput1
- .join(joinInput2, JoinHint.REPARTITION_HASH_FIRST).where(0).equalTo(1)
- .with(new DummyFlatJoinFunction<Tuple2<Long,Long>>());
-
- input
- .map(new IdentityMapper<Tuple2<Long,Long>>())
- .withBroadcastSet(bc_input1, "bc1")
- .union(joinResult)
- .print();
-
- Plan plan = env.createProgramPlan();
- compileNoStats(plan);
- }
- catch(Exception e){
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- private static final class Duplicator<T> implements MapFunction<T, Tuple2<T, T>> {
-
- @Override
- public Tuple2<T, T> map(T value) {
- return new Tuple2<T, T>(value, value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
deleted file mode 100644
index c7ad2da..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-
-@SuppressWarnings("serial")
-public class BroadcastVariablePipelinebreakerTest extends CompilerTestBase {
-
- @Test
- public void testNoBreakerForIndependentVariable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<String> source1 = env.fromElements("test");
- DataSet<String> source2 = env.fromElements("test");
-
- source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name").print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
-
- assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testBreakerForDependentVariable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<String> source1 = env.fromElements("test");
-
- source1.map(new IdentityMapper<String>()).map(new IdentityMapper<String>()).withBroadcastSet(source1, "some name").print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
-
- assertEquals(TempMode.PIPELINE_BREAKER, mapper.getInput().getTempMode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
deleted file mode 100644
index 3e7da6c..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.common.functions.RichJoinFunction;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
-* Tests that validate optimizer choice when using hash joins inside of iterations
-*/
-@SuppressWarnings("serial")
-public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
-
- /**
- * This tests whether a HYBRIDHASH_BUILD_SECOND is correctly transformed to a HYBRIDHASH_BUILD_SECOND_CACHED
- * when inside of an iteration an on the static path
- */
- @Test
- public void testRightSide() {
- try {
-
- Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
-
- // verify correct join strategy
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
- assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
- assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- /**
- * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
- */
- @Test
- public void testRightSideCountercheck() {
- try {
-
- Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
-
- // verify correct join strategy
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
- assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
- assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- /**
- * This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
- * when inside of an iteration an on the static path
- */
- @Test
- public void testLeftSide() {
- try {
-
- Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
-
- // verify correct join strategy
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
- assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
- assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- /**
- * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
- */
- @Test
- public void testLeftSideCountercheck() {
- try {
-
- Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
-
- // verify correct join strategy
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
- assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
- assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- /**
- * This test simulates a join of a big left side with a small right side inside of an iteration, where the small side is on a static path.
- * Currently the best execution plan is a HYBRIDHASH_BUILD_SECOND_CACHED, where the small side is hashed and cached.
- * This test also makes sure that all relevant plans are correctly enumerated by the optimizer.
- */
- @Test
- public void testCorrectChoosing() {
- try {
-
- Plan plan = getTestPlanRightStatic("");
-
- SourceCollectorVisitor sourceCollector = new SourceCollectorVisitor();
- plan.accept(sourceCollector);
-
- for(GenericDataSourceBase<?, ?> s : sourceCollector.getSources()) {
- if(s.getName().equals("bigFile")) {
- this.setSourceStatistics(s, 10000000, 1000);
- }
- else if(s.getName().equals("smallFile")) {
- this.setSourceStatistics(s, 100, 100);
- }
- }
-
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
-
- // verify correct join strategy
- assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
- assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
- assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
- catch (Exception e) {
- System.err.println(e.getMessage());
- e.printStackTrace();
- fail("Test errored: " + e.getMessage());
- }
- }
-
- private Plan getTestPlanRightStatic(String strategy) {
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSet<Tuple3<Long, Long, Long>> bigInput = env.readCsvFile("file://bigFile").types(Long.class, Long.class, Long.class).name("bigFile");
-
- DataSet<Tuple3<Long, Long, Long>> smallInput = env.readCsvFile("file://smallFile").types(Long.class, Long.class, Long.class).name("smallFile");
-
- IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
-
- Configuration joinStrategy = new Configuration();
- joinStrategy.setString(Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
-
- if(strategy != "") {
- joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
- }
-
- DataSet<Tuple3<Long, Long, Long>> inner = iteration.join(smallInput).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
-
- DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
-
- output.print();
-
- return env.createProgramPlan();
-
- }
-
- private Plan getTestPlanLeftStatic(String strategy) {
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> bigInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L),
- new Tuple3<Long, Long, Long>(1L, 2L, 3L),new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Big");
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> smallInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Small");
-
- IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
-
- Configuration joinStrategy = new Configuration();
- joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
-
- DataSet<Tuple3<Long, Long, Long>> inner = smallInput.join(iteration).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
-
- DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
-
- output.print();
-
- return env.createProgramPlan();
-
- }
-
- private static class DummyJoiner extends RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> {
-
- @Override
- public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first,
- Tuple3<Long, Long, Long> second) throws Exception {
-
- return first;
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
deleted file mode 100644
index eba07f1..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.functions.RichCoGroupFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.Visitor;
-
-@SuppressWarnings("serial")
-public class CoGroupSolutionSetFirstTest extends CompilerTestBase {
-
- public static class SimpleCGroup extends RichCoGroupFunction<Tuple1<Integer>, Tuple1<Integer>, Tuple1<Integer>> {
- @Override
- public void coGroup(Iterable<Tuple1<Integer>> first, Iterable<Tuple1<Integer>> second, Collector<Tuple1<Integer>> out) {}
- }
-
- public static class SimpleMap extends RichMapFunction<Tuple1<Integer>, Tuple1<Integer>> {
- @Override
- public Tuple1<Integer> map(Tuple1<Integer> value) throws Exception {
- return null;
- }
- }
-
- @Test
- public void testCoGroupSolutionSet() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple1<Integer>> raw = env.readCsvFile(IN_FILE).types(Integer.class);
-
- DeltaIteration<Tuple1<Integer>, Tuple1<Integer>> iteration = raw.iterateDelta(raw, 1000, 0);
-
- DataSet<Tuple1<Integer>> test = iteration.getWorkset().map(new SimpleMap());
- DataSet<Tuple1<Integer>> delta = iteration.getSolutionSet().coGroup(test).where(0).equalTo(0).with(new SimpleCGroup());
- DataSet<Tuple1<Integer>> feedback = iteration.getWorkset().map(new SimpleMap());
- DataSet<Tuple1<Integer>> result = iteration.closeWith(delta, feedback);
-
- result.print();
-
- Plan plan = env.createProgramPlan();
- OptimizedPlan oPlan = null;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException e) {
- Assert.fail(e.getMessage());
- }
-
- oPlan.accept(new Visitor<PlanNode>() {
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof WorksetIterationPlanNode) {
- PlanNode deltaNode = ((WorksetIterationPlanNode) visitable).getSolutionSetDeltaPlanNode();
-
- //get the CoGroup
- DualInputPlanNode dpn = (DualInputPlanNode) deltaNode.getInputs().iterator().next().getSource();
- Channel in1 = dpn.getInput1();
- Channel in2 = dpn.getInput2();
-
- Assert.assertTrue(in1.getLocalProperties().getOrdering() == null);
- Assert.assertTrue(in2.getLocalProperties().getOrdering() != null);
- Assert.assertTrue(in2.getLocalProperties().getOrdering().getInvolvedIndexes().contains(0));
- Assert.assertTrue(in1.getShipStrategy() == ShipStrategyType.FORWARD);
- Assert.assertTrue(in2.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {}
- });
- }
-}
[10/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
new file mode 100644
index 0000000..2d8377e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/GenericFlatTypePostPass.java
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.postpass;
+
+import java.util.Map;
+
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerPostPassException;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+
+/**
+ *
+ */
+public abstract class GenericFlatTypePostPass<X, T extends AbstractSchema<X>> implements OptimizerPostPass {
+
+ private boolean propagateParentSchemaDown = true;
+
+
+ public boolean isPropagateParentSchemaDown() {
+ return propagateParentSchemaDown;
+ }
+
+ public void setPropagateParentSchemaDown(boolean propagateParentSchemaDown) {
+ this.propagateParentSchemaDown = propagateParentSchemaDown;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Generic schema inferring traversal
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void postPass(OptimizedPlan plan) {
+ for (SinkPlanNode sink : plan.getDataSinks()) {
+ traverse(sink, null, true);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected void traverse(PlanNode node, T parentSchema, boolean createUtilities) {
+ // distinguish the node types
+ if (node instanceof SinkPlanNode) {
+ SinkPlanNode sn = (SinkPlanNode) node;
+ Channel inchannel = sn.getInput();
+
+ T schema = createEmptySchema();
+ sn.postPassHelper = schema;
+
+ // add the sinks information to the schema
+ try {
+ getSinkSchema(sn, schema);
+ }
+ catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Conflicting type infomation for the data sink '" +
+ sn.getSinkNode().getOperator().getName() + "'.");
+ }
+
+ // descend to the input channel
+ try {
+ propagateToChannel(schema, inchannel, createUtilities);
+ }
+ catch (MissingFieldTypeInfoException ex) {
+ throw new CompilerPostPassException("Missing type infomation for the channel that inputs to the data sink '" +
+ sn.getSinkNode().getOperator().getName() + "'.");
+ }
+ }
+ else if (node instanceof SourcePlanNode) {
+ if (createUtilities) {
+ ((SourcePlanNode) node).setSerializer(createSerializer(parentSchema, node));
+ // nothing else to be done here. the source has no input and no strategy itself
+ }
+ }
+ else if (node instanceof BulkIterationPlanNode) {
+ BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
+
+ // get the nodes current schema
+ T schema;
+ if (iterationNode.postPassHelper == null) {
+ schema = createEmptySchema();
+ iterationNode.postPassHelper = schema;
+ } else {
+ schema = (T) iterationNode.postPassHelper;
+ }
+ schema.increaseNumConnectionsThatContributed();
+
+ // add the parent schema to the schema
+ if (propagateParentSchemaDown) {
+ addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
+ }
+
+ // check whether all outgoing channels have not yet contributed. come back later if not.
+ if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
+ return;
+ }
+
+ if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
+ }
+
+ // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
+ if (iterationNode.getRootOfTerminationCriterion() != null) {
+ SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
+ traverse(addMapper.getInput().getSource(), createEmptySchema(), false);
+ try {
+ addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
+ } catch (MissingFieldTypeInfoException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // traverse the step function for the first time. create schema only, no utilities
+ traverse(iterationNode.getRootOfStepFunction(), schema, false);
+
+ T pss = (T) iterationNode.getPartialSolutionPlanNode().postPassHelper;
+ if (pss == null) {
+ throw new CompilerException("Error in Optimizer Post Pass: Partial solution schema is null after first traversal of the step function.");
+ }
+
+ // traverse the step function for the second time, taking the schema of the partial solution
+ traverse(iterationNode.getRootOfStepFunction(), pss, createUtilities);
+
+ if (iterationNode.getRootOfTerminationCriterion() != null) {
+ SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
+ traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities);
+ try {
+ addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
+ } catch (MissingFieldTypeInfoException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // take the schema from the partial solution node and add its fields to the iteration result schema.
+ // input and output schema need to be identical, so this is essentially a sanity check
+ addSchemaToSchema(pss, schema, iterationNode.getProgramOperator().getName());
+
+ // set the serializer
+ if (createUtilities) {
+ iterationNode.setSerializerForIterationChannel(createSerializer(pss, iterationNode.getPartialSolutionPlanNode()));
+ }
+
+ // done, we can now propagate our info down
+ try {
+ propagateToChannel(schema, iterationNode.getInput(), createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
+ + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
+ e.getFieldNumber());
+ }
+ }
+ else if (node instanceof WorksetIterationPlanNode) {
+ WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
+
+ // get the nodes current schema
+ T schema;
+ if (iterationNode.postPassHelper == null) {
+ schema = createEmptySchema();
+ iterationNode.postPassHelper = schema;
+ } else {
+ schema = (T) iterationNode.postPassHelper;
+ }
+ schema.increaseNumConnectionsThatContributed();
+
+ // add the parent schema to the schema (which refers to the solution set schema)
+ if (propagateParentSchemaDown) {
+ addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName());
+ }
+
+ // check whether all outgoing channels have not yet contributed. come back later if not.
+ if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
+ return;
+ }
+ if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
+ }
+ if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
+ }
+
+ // traverse the step function
+ // pass an empty schema to the next workset and the parent schema to the solution set delta
+ // these first traversals are schema only
+ traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false);
+ traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false);
+
+ T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper;
+ T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper;
+
+ if (wss == null) {
+ throw new CompilerException("Error in Optimizer Post Pass: Workset schema is null after first traversal of the step function.");
+ }
+ if (sss == null) {
+ throw new CompilerException("Error in Optimizer Post Pass: Solution set schema is null after first traversal of the step function.");
+ }
+
+ // make the second pass and instantiate the utilities
+ traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities);
+ traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities);
+
+ // add the types from the solution set schema to the iteration's own schema. since
+ // the solution set input and the result must have the same schema, this acts as a sanity check.
+ try {
+ for (Map.Entry<Integer, X> entry : sss) {
+ Integer pos = entry.getKey();
+ schema.addType(pos, entry.getValue());
+ }
+ } catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
+ + " in node '" + iterationNode.getProgramOperator().getName() + "'. Contradicting types between the " +
+ "result of the iteration and the solution set schema: " + e.getPreviousType() +
+ " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
+ }
+
+ // set the serializers and comparators
+ if (createUtilities) {
+ WorksetIterationNode optNode = iterationNode.getIterationNode();
+ iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
+ iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
+ try {
+ iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
+ } catch (MissingFieldTypeInfoException ex) {
+ throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
+ optNode.getOperator().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
+ }
+ }
+
+ // done, we can now propagate our info down
+ try {
+ propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
+ propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
+ } catch (MissingFieldTypeInfoException ex) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
+ + iterationNode.getProgramOperator().getName() + "'. Missing type information for key field " +
+ ex.getFieldNumber());
+ }
+ }
+ else if (node instanceof SingleInputPlanNode) {
+ SingleInputPlanNode sn = (SingleInputPlanNode) node;
+
+ // get the nodes current schema
+ T schema;
+ if (sn.postPassHelper == null) {
+ schema = createEmptySchema();
+ sn.postPassHelper = schema;
+ } else {
+ schema = (T) sn.postPassHelper;
+ }
+ schema.increaseNumConnectionsThatContributed();
+ SingleInputNode optNode = sn.getSingleInputNode();
+
+ // add the parent schema to the schema
+ if (propagateParentSchemaDown) {
+ addSchemaToSchema(parentSchema, schema, optNode, 0);
+ }
+
+ // check whether all outgoing channels have not yet contributed. come back later if not.
+ if (schema.getNumConnectionsThatContributed() < sn.getOutgoingChannels().size()) {
+ return;
+ }
+
+ // add the nodes local information
+ try {
+ getSingleInputNodeSchema(sn, schema);
+ } catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
+ }
+
+ if (createUtilities) {
+ // parameterize the node's driver strategy
+ for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
+ try {
+ sn.setComparator(createComparator(sn.getKeys(i), sn.getSortOrders(i), schema),i);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
+ optNode.getOperator().getName() + "'. Missing type information for key field " +
+ e.getFieldNumber());
+ }
+ }
+ }
+
+ // done, we can now propagate our info down
+ try {
+ propagateToChannel(schema, sn.getInput(), createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" +
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+
+ // don't forget the broadcast inputs
+ for (Channel c: sn.getBroadcastInputs()) {
+ try {
+ propagateToChannel(createEmptySchema(), c, createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+ }
+ }
+ else if (node instanceof DualInputPlanNode) {
+ DualInputPlanNode dn = (DualInputPlanNode) node;
+
+ // get the nodes current schema
+ T schema1;
+ T schema2;
+ if (dn.postPassHelper1 == null) {
+ schema1 = createEmptySchema();
+ schema2 = createEmptySchema();
+ dn.postPassHelper1 = schema1;
+ dn.postPassHelper2 = schema2;
+ } else {
+ schema1 = (T) dn.postPassHelper1;
+ schema2 = (T) dn.postPassHelper2;
+ }
+
+ schema1.increaseNumConnectionsThatContributed();
+ schema2.increaseNumConnectionsThatContributed();
+ TwoInputNode optNode = dn.getTwoInputNode();
+
+ // add the parent schema to the schema
+ if (propagateParentSchemaDown) {
+ addSchemaToSchema(parentSchema, schema1, optNode, 0);
+ addSchemaToSchema(parentSchema, schema2, optNode, 1);
+ }
+
+ // check whether all outgoing channels have not yet contributed. come back later if not.
+ if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
+ return;
+ }
+
+ // add the nodes local information
+ try {
+ getDualInputNodeSchema(dn, schema1, schema2);
+ } catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getOperator().getName()));
+ }
+
+ // parameterize the node's driver strategy
+ if (createUtilities) {
+ if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
+ // set the individual comparators
+ try {
+ dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
+ dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+
+ // set the pair comparator
+ try {
+ dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(),
+ dn.getSortOrders(), schema1, schema2));
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+
+ }
+ }
+
+ // done, we can now propagate our info down
+ try {
+ propagateToChannel(schema1, dn.getInput1(), createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
+ + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+ try {
+ propagateToChannel(schema2, dn.getInput2(), createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
+ + optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+
+ // don't forget the broadcast inputs
+ for (Channel c: dn.getBroadcastInputs()) {
+ try {
+ propagateToChannel(createEmptySchema(), c, createUtilities);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
+ optNode.getOperator().getName() + "'. Missing type information for field " + e.getFieldNumber());
+ }
+ }
+ }
+ else if (node instanceof NAryUnionPlanNode) {
+ // only propagate the info down
+ try {
+ for (Channel channel : node.getInputs()) {
+ propagateToChannel(parentSchema, channel, createUtilities);
+ }
+ } catch (MissingFieldTypeInfoException ex) {
+ throw new CompilerPostPassException("Could not set up runtime strategy for the input channel to " +
+ " a union node. Missing type information for field " + ex.getFieldNumber());
+ }
+ }
+ // catch the sources of the iterative step functions
+ else if (node instanceof BulkPartialSolutionPlanNode ||
+ node instanceof SolutionSetPlanNode ||
+ node instanceof WorksetPlanNode)
+ {
+ // get the nodes current schema
+ T schema;
+ String name;
+ if (node instanceof BulkPartialSolutionPlanNode) {
+ BulkPartialSolutionPlanNode psn = (BulkPartialSolutionPlanNode) node;
+ if (psn.postPassHelper == null) {
+ schema = createEmptySchema();
+ psn.postPassHelper = schema;
+ } else {
+ schema = (T) psn.postPassHelper;
+ }
+ name = "partial solution of bulk iteration '" +
+ psn.getPartialSolutionNode().getIterationNode().getOperator().getName() + "'";
+ }
+ else if (node instanceof SolutionSetPlanNode) {
+ SolutionSetPlanNode ssn = (SolutionSetPlanNode) node;
+ if (ssn.postPassHelper == null) {
+ schema = createEmptySchema();
+ ssn.postPassHelper = schema;
+ } else {
+ schema = (T) ssn.postPassHelper;
+ }
+ name = "solution set of workset iteration '" +
+ ssn.getSolutionSetNode().getIterationNode().getOperator().getName() + "'";
+ }
+ else if (node instanceof WorksetPlanNode) {
+ WorksetPlanNode wsn = (WorksetPlanNode) node;
+ if (wsn.postPassHelper == null) {
+ schema = createEmptySchema();
+ wsn.postPassHelper = schema;
+ } else {
+ schema = (T) wsn.postPassHelper;
+ }
+ name = "workset of workset iteration '" +
+ wsn.getWorksetNode().getIterationNode().getOperator().getName() + "'";
+ } else {
+ throw new CompilerException();
+ }
+
+ schema.increaseNumConnectionsThatContributed();
+
+ // add the parent schema to the schema
+ addSchemaToSchema(parentSchema, schema, name);
+ }
+ else {
+ throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
+ }
+ }
+
+ private void propagateToChannel(T schema, Channel channel, boolean createUtilities) throws MissingFieldTypeInfoException {
+ if (createUtilities) {
+ // the serializer always exists
+ channel.setSerializer(createSerializer(schema));
+
+ // parameterize the ship strategy
+ if (channel.getShipStrategy().requiresComparator()) {
+ channel.setShipStrategyComparator(
+ createComparator(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder(), schema));
+ }
+
+ // parameterize the local strategy
+ if (channel.getLocalStrategy().requiresComparator()) {
+ channel.setLocalStrategyComparator(
+ createComparator(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder(), schema));
+ }
+ }
+
+ // propagate the channel's source model
+ traverse(channel.getSource(), schema, createUtilities);
+ }
+
+ private void addSchemaToSchema(T sourceSchema, T targetSchema, String opName) {
+ try {
+ for (Map.Entry<Integer, X> entry : sourceSchema) {
+ Integer pos = entry.getKey();
+ targetSchema.addType(pos, entry.getValue());
+ }
+ } catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
+ + " in node '" + opName + "' propagated from successor node. " +
+ "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() +
+ ". Most probable cause: Invalid constant field annotations.");
+ }
+ }
+
+ private void addSchemaToSchema(T sourceSchema, T targetSchema, OptimizerNode optNode, int input) {
+ try {
+ for (Map.Entry<Integer, X> entry : sourceSchema) {
+ Integer pos = entry.getKey();
+ SemanticProperties sprops = optNode.getSemanticProperties();
+
+ if (sprops != null && sprops.getForwardingTargetFields(input, pos) != null && sprops.getForwardingTargetFields(input, pos).contains(pos)) {
+ targetSchema.addType(pos, entry.getValue());
+ }
+ }
+ } catch (ConflictingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
+ + " in node '" + optNode.getOperator().getName() + "' propagated from successor node. " +
+ "Conflicting types: " + e.getPreviousType() + " and " + e.getNewType() +
+ ". Most probable cause: Invalid constant field annotations.");
+ }
+ }
+
+ private String getConflictingTypeErrorMessage(ConflictingFieldTypeInfoException e, String operatorName) {
+ return "Conflicting type information for field " + e.getFieldNumber()
+ + " in node '" + operatorName + "' between types declared in the node's "
+ + "contract and types inferred from successor contracts. Conflicting types: "
+ + e.getPreviousType() + " and " + e.getNewType()
+ + ". Most probable cause: Invalid constant field annotations.";
+ }
+
+ private TypeSerializerFactory<?> createSerializer(T schema, PlanNode node) {
+ try {
+ return createSerializer(schema);
+ } catch (MissingFieldTypeInfoException e) {
+ throw new CompilerPostPassException("Missing type information while creating serializer for '" +
+ node.getProgramOperator().getName() + "'.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Type specific methods that extract schema information
+ // --------------------------------------------------------------------------------------------
+
+ protected abstract T createEmptySchema();
+
+ protected abstract void getSinkSchema(SinkPlanNode sink, T schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
+
+ protected abstract void getSingleInputNodeSchema(SingleInputPlanNode node, T schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
+
+ protected abstract void getDualInputNodeSchema(DualInputPlanNode node, T input1Schema, T input2Schema) throws CompilerPostPassException, ConflictingFieldTypeInfoException;
+
+ // --------------------------------------------------------------------------------------------
+ // Methods to create serializers and comparators
+ // --------------------------------------------------------------------------------------------
+
+ protected abstract TypeSerializerFactory<?> createSerializer(T schema) throws MissingFieldTypeInfoException;
+
+ protected abstract TypeComparatorFactory<?> createComparator(FieldList fields, boolean[] directions, T schema) throws MissingFieldTypeInfoException;
+
+ protected abstract TypePairComparatorFactory<?, ?> createPairComparator(FieldList fields1, FieldList fields2, boolean[] sortDirections,
+ T schema1, T schema2) throws MissingFieldTypeInfoException;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
new file mode 100644
index 0000000..5fdf3dd
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/JavaApiPostPass.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.operators.DualInputOperator;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeinfo.AtomicType;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.TypeComparator;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.runtime.RuntimeComparatorFactory;
+import org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory;
+import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerPostPassException;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.optimizer.util.NoOpUnaryUdfOp;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ * The post-optimizer plan traversal. This traversal fills in the API specific utilities (serializers and
+ * comparators).
+ */
+public class JavaApiPostPass implements OptimizerPostPass {
+
+ private final Set<PlanNode> alreadyDone = new HashSet<PlanNode>();
+
+ private ExecutionConfig executionConfig = null;
+
+ @Override
+ public void postPass(OptimizedPlan plan) {
+
+ executionConfig = plan.getOriginalPactPlan().getExecutionConfig();
+
+ for (SinkPlanNode sink : plan.getDataSinks()) {
+ traverse(sink);
+ }
+ }
+
+
+ protected void traverse(PlanNode node) {
+ if (!alreadyDone.add(node)) {
+ // already worked on that one
+ return;
+ }
+
+ // distinguish the node types
+ if (node instanceof SinkPlanNode) {
+ // descend to the input channel
+ SinkPlanNode sn = (SinkPlanNode) node;
+ Channel inchannel = sn.getInput();
+ traverseChannel(inchannel);
+ }
+ else if (node instanceof SourcePlanNode) {
+ TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
+ ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
+ }
+ else if (node instanceof BulkIterationPlanNode) {
+ BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
+
+ if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
+ }
+
+ // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
+ if (iterationNode.getRootOfTerminationCriterion() != null) {
+ SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
+ traverseChannel(addMapper.getInput());
+ }
+
+ BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator();
+
+ // set the serializer
+ iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));
+
+ // done, we can now propagate our info down
+ traverseChannel(iterationNode.getInput());
+ traverse(iterationNode.getRootOfStepFunction());
+ }
+ else if (node instanceof WorksetIterationPlanNode) {
+ WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
+
+ if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
+ }
+ if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
+ throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
+ }
+
+ DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator();
+
+ // set the serializers and comparators for the workset iteration
+ iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
+ iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
+ iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
+ iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
+
+ // traverse the inputs
+ traverseChannel(iterationNode.getInput1());
+ traverseChannel(iterationNode.getInput2());
+
+ // traverse the step function
+ traverse(iterationNode.getSolutionSetDeltaPlanNode());
+ traverse(iterationNode.getNextWorkSetPlanNode());
+ }
+ else if (node instanceof SingleInputPlanNode) {
+ SingleInputPlanNode sn = (SingleInputPlanNode) node;
+
+ if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) {
+
+ // Special case for delta iterations
+ if(sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) {
+ traverseChannel(sn.getInput());
+ return;
+ } else {
+ throw new RuntimeException("Wrong operator type found in post pass.");
+ }
+ }
+
+ SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator();
+
+ // parameterize the node's driver strategy
+ for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
+ sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i),
+ getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
+ }
+ // done, we can now propagate our info down
+ traverseChannel(sn.getInput());
+
+ // don't forget the broadcast inputs
+ for (Channel c: sn.getBroadcastInputs()) {
+ traverseChannel(c);
+ }
+ }
+ else if (node instanceof DualInputPlanNode) {
+ DualInputPlanNode dn = (DualInputPlanNode) node;
+
+ if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) {
+ throw new RuntimeException("Wrong operator type found in post pass.");
+ }
+
+ DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator();
+
+ // parameterize the node's driver strategy
+ if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
+ dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
+ getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
+ dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
+ getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));
+
+ dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
+ dualInputOperator.getOperatorInfo().getSecondInputType()));
+
+ }
+
+ traverseChannel(dn.getInput1());
+ traverseChannel(dn.getInput2());
+
+ // don't forget the broadcast inputs
+ for (Channel c: dn.getBroadcastInputs()) {
+ traverseChannel(c);
+ }
+
+ }
+ // catch the sources of the iterative step functions
+ else if (node instanceof BulkPartialSolutionPlanNode ||
+ node instanceof SolutionSetPlanNode ||
+ node instanceof WorksetPlanNode)
+ {
+ // Do nothing :D
+ }
+ else if (node instanceof NAryUnionPlanNode){
+ // Traverse to all child channels
+ for (Channel channel : node.getInputs()) {
+ traverseChannel(channel);
+ }
+ }
+ else {
+ throw new CompilerPostPassException("Unknown node type encountered: " + node.getClass().getName());
+ }
+ }
+
+ private void traverseChannel(Channel channel) {
+
+ PlanNode source = channel.getSource();
+ Operator<?> javaOp = source.getProgramOperator();
+
+// if (!(javaOp instanceof BulkIteration) && !(javaOp instanceof JavaPlanNode)) {
+// throw new RuntimeException("Wrong operator type found in post pass: " + javaOp);
+// }
+
+ TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
+
+
+ if(javaOp instanceof GroupReduceOperatorBase &&
+ (source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_REDUCE_COMBINE)) {
+ GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp;
+ type = groupNode.getInput().getOperatorInfo().getOutputType();
+ }
+ else if(javaOp instanceof PlanUnwrappingReduceGroupOperator &&
+ source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) {
+ PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp;
+ type = groupNode.getInput().getOperatorInfo().getOutputType();
+ }
+
+ // the serializer always exists
+ channel.setSerializer(createSerializer(type));
+
+ // parameterize the ship strategy
+ if (channel.getShipStrategy().requiresComparator()) {
+ channel.setShipStrategyComparator(createComparator(type, channel.getShipStrategyKeys(),
+ getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder())));
+ }
+
+ // parameterize the local strategy
+ if (channel.getLocalStrategy().requiresComparator()) {
+ channel.setLocalStrategyComparator(createComparator(type, channel.getLocalStrategyKeys(),
+ getSortOrders(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder())));
+ }
+
+ // descend to the channel's source
+ traverse(channel.getSource());
+ }
+
+
+ @SuppressWarnings("unchecked")
+ private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) {
+ Operator<?> op = node.getOptimizerNode().getOperator();
+
+ if (op instanceof GenericDataSourceBase) {
+ return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType();
+ } else {
+ throw new RuntimeException("Wrong operator type found in post pass.");
+ }
+ }
+
+ private <T> TypeSerializerFactory<?> createSerializer(TypeInformation<T> typeInfo) {
+ TypeSerializer<T> serializer = typeInfo.createSerializer(executionConfig);
+
+ return new RuntimeSerializerFactory<T>(serializer, typeInfo.getTypeClass());
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
+
+ TypeComparator<T> comparator;
+ if (typeInfo instanceof CompositeType) {
+ comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
+ }
+ else if (typeInfo instanceof AtomicType) {
+ // handle grouping of atomic types
+ comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
+ }
+ else {
+ throw new RuntimeException("Unrecognized type: " + typeInfo);
+ }
+
+ return new RuntimeComparatorFactory<T>(comparator);
+ }
+
+ private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
+ if (!(typeInfo1.isTupleType() || typeInfo1 instanceof PojoTypeInfo) && (typeInfo2.isTupleType() || typeInfo2 instanceof PojoTypeInfo)) {
+ throw new RuntimeException("The runtime currently supports only keyed binary operations (such as joins) on tuples and POJO types.");
+ }
+
+// @SuppressWarnings("unchecked")
+// TupleTypeInfo<T1> info1 = (TupleTypeInfo<T1>) typeInfo1;
+// @SuppressWarnings("unchecked")
+// TupleTypeInfo<T2> info2 = (TupleTypeInfo<T2>) typeInfo2;
+
+ return new RuntimePairComparatorFactory<T1,T2>();
+ }
+
+ private static final boolean[] getSortOrders(FieldList keys, boolean[] orders) {
+ if (orders == null) {
+ orders = new boolean[keys.size()];
+ Arrays.fill(orders, true);
+ }
+ return orders;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
new file mode 100644
index 0000000..b9f6bfa
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+public final class MissingFieldTypeInfoException extends Exception {
+
+ private static final long serialVersionUID = 8749941961302509358L;
+
+ private final int fieldNumber;
+
+ public MissingFieldTypeInfoException(int fieldNumber) {
+ this.fieldNumber = fieldNumber;
+ }
+
+ public int getFieldNumber() {
+ return fieldNumber;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
new file mode 100644
index 0000000..ba0b7c7
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/OptimizerPostPass.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.postpass;
+
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+
+/**
+ * Interface for visitors that process the optimizer's plan. Typical post processing applications are schema
+ * finalization or the generation/parameterization of utilities for the actual data model.
+ */
+public interface OptimizerPostPass {
+
+ /**
+ * Central post processing function. Invoked by the optimizer after the best plan has
+ * been determined.
+ *
+ * @param plan The plan to be post processed.
+ */
+ void postPass(OptimizedPlan plan);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
new file mode 100644
index 0000000..1fc4c34
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/PostPassUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.types.Key;
+
+
+public class PostPassUtils {
+
+ public static <X> Class<? extends Key<?>>[] getKeys(AbstractSchema<Class< ? extends X>> schema, int[] fields) throws MissingFieldTypeInfoException {
+ @SuppressWarnings("unchecked")
+ Class<? extends Key<?>>[] keyTypes = new Class[fields.length];
+
+ for (int i = 0; i < fields.length; i++) {
+ Class<? extends X> type = schema.getType(fields[i]);
+ if (type == null) {
+ throw new MissingFieldTypeInfoException(i);
+ } else if (Key.class.isAssignableFrom(type)) {
+ @SuppressWarnings("unchecked")
+ Class<? extends Key<?>> keyType = (Class<? extends Key<?>>) type;
+ keyTypes[i] = keyType;
+ } else {
+ throw new CompilerException("The field type " + type.getName() +
+ " cannot be used as a key because it does not implement the interface 'Key'");
+ }
+ }
+
+ return keyTypes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
new file mode 100644
index 0000000..8a2d006
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/RecordModelPostPass.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+import org.apache.flink.api.common.operators.DualInputOperator;
+import org.apache.flink.api.common.operators.GenericDataSinkBase;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.RecordOperator;
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.api.common.typeutils.record.RecordComparatorFactory;
+import org.apache.flink.api.common.typeutils.record.RecordPairComparatorFactory;
+import org.apache.flink.api.common.typeutils.record.RecordSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerPostPassException;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.types.Key;
+
+/**
+ * Post pass implementation for the Record data model. Does only type inference and creates
+ * serializers and comparators.
+ */
+public class RecordModelPostPass extends GenericFlatTypePostPass<Class<? extends Key<?>>, SparseKeySchema> {
+
+ // --------------------------------------------------------------------------------------------
+ // Type specific methods that extract schema information
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected SparseKeySchema createEmptySchema() {
+ return new SparseKeySchema();
+ }
+
+ @Override
+ protected void getSinkSchema(SinkPlanNode sinkPlanNode, SparseKeySchema schema) throws CompilerPostPassException {
+ GenericDataSinkBase<?> sink = sinkPlanNode.getSinkNode().getOperator();
+ Ordering partitioning = sink.getPartitionOrdering();
+ Ordering sorting = sink.getLocalOrder();
+
+ try {
+ if (partitioning != null) {
+ addOrderingToSchema(partitioning, schema);
+ }
+ if (sorting != null) {
+ addOrderingToSchema(sorting, schema);
+ }
+ } catch (ConflictingFieldTypeInfoException ex) {
+ throw new CompilerPostPassException("Conflicting information found when adding data sink types. " +
+ "Probable reason is contradicting type infos for partitioning and sorting ordering.");
+ }
+ }
+
+ @Override
+ protected void getSingleInputNodeSchema(SingleInputPlanNode node, SparseKeySchema schema)
+ throws CompilerPostPassException, ConflictingFieldTypeInfoException
+ {
+ // check that we got the right types
+ SingleInputOperator<?, ?, ?> contract = (SingleInputOperator<?, ?, ?>) node.getSingleInputNode().getOperator();
+ if (! (contract instanceof RecordOperator)) {
+ throw new CompilerPostPassException("Error: Operator is not a Record based contract. Wrong compiler invokation.");
+ }
+ RecordOperator recContract = (RecordOperator) contract;
+
+ // add the information to the schema
+ int[] localPositions = contract.getKeyColumns(0);
+ Class<? extends Key<?>>[] types = recContract.getKeyClasses();
+ for (int i = 0; i < localPositions.length; i++) {
+ schema.addType(localPositions[i], types[i]);
+ }
+
+ // this is a temporary fix, we should solve this more generic
+ if (contract instanceof GroupReduceOperatorBase) {
+ Ordering groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) contract).getGroupOrder();
+ if (groupOrder != null) {
+ addOrderingToSchema(groupOrder, schema);
+ }
+ }
+ }
+
+ @Override
+ protected void getDualInputNodeSchema(DualInputPlanNode node, SparseKeySchema input1Schema, SparseKeySchema input2Schema)
+ throws CompilerPostPassException, ConflictingFieldTypeInfoException
+ {
+ // add the nodes local information. this automatically consistency checks
+ DualInputOperator<?, ?, ?, ?> contract = node.getTwoInputNode().getOperator();
+ if (! (contract instanceof RecordOperator)) {
+ throw new CompilerPostPassException("Error: Operator is not a Pact Record based contract. Wrong compiler invokation.");
+ }
+
+ RecordOperator recContract = (RecordOperator) contract;
+ int[] localPositions1 = contract.getKeyColumns(0);
+ int[] localPositions2 = contract.getKeyColumns(1);
+ Class<? extends Key<?>>[] types = recContract.getKeyClasses();
+
+ if (localPositions1.length != localPositions2.length) {
+ throw new CompilerException("Error: The keys for the first and second input have a different number of fields.");
+ }
+
+ for (int i = 0; i < localPositions1.length; i++) {
+ input1Schema.addType(localPositions1[i], types[i]);
+ }
+ for (int i = 0; i < localPositions2.length; i++) {
+ input2Schema.addType(localPositions2[i], types[i]);
+ }
+
+
+ // this is a temporary fix, we should solve this more generic
+ if (contract instanceof CoGroupOperatorBase) {
+ Ordering groupOrder1 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputOne();
+ Ordering groupOrder2 = ((CoGroupOperatorBase<?, ?, ?, ?>) contract).getGroupOrderForInputTwo();
+
+ if (groupOrder1 != null) {
+ addOrderingToSchema(groupOrder1, input1Schema);
+ }
+ if (groupOrder2 != null) {
+ addOrderingToSchema(groupOrder2, input2Schema);
+ }
+ }
+ }
+
+ private void addOrderingToSchema(Ordering o, SparseKeySchema schema) throws ConflictingFieldTypeInfoException {
+ for (int i = 0; i < o.getNumberOfFields(); i++) {
+ Integer pos = o.getFieldNumber(i);
+ Class<? extends Key<?>> type = o.getType(i);
+ schema.addType(pos, type);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Methods to create serializers and comparators
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected TypeSerializerFactory<?> createSerializer(SparseKeySchema schema) {
+ return RecordSerializerFactory.get();
+ }
+
+ @Override
+ protected RecordComparatorFactory createComparator(FieldList fields, boolean[] directions, SparseKeySchema schema)
+ throws MissingFieldTypeInfoException
+ {
+ int[] positions = fields.toArray();
+ Class<? extends Key<?>>[] keyTypes = PostPassUtils.getKeys(schema, positions);
+ return new RecordComparatorFactory(positions, keyTypes, directions);
+ }
+
+ @Override
+ protected RecordPairComparatorFactory createPairComparator(FieldList fields1, FieldList fields2, boolean[] sortDirections,
+ SparseKeySchema schema1, SparseKeySchema schema2)
+ {
+ return RecordPairComparatorFactory.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
new file mode 100644
index 0000000..e14888e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.flink.types.Key;
+
+/**
+ * Class encapsulating a schema map (int column position -> column type) and a reference counter.
+ */
+public class SparseKeySchema extends AbstractSchema<Class<? extends Key<?>>> {
+
+ private final Map<Integer, Class<? extends Key<?>>> schema;
+
+
+ public SparseKeySchema() {
+ this.schema = new HashMap<Integer, Class<? extends Key<?>>>();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void addType(int key, Class<? extends Key<?>> type) throws ConflictingFieldTypeInfoException {
+ Class<? extends Key<?>> previous = this.schema.put(key, type);
+ if (previous != null && previous != type) {
+ throw new ConflictingFieldTypeInfoException(key, previous, type);
+ }
+ }
+
+ @Override
+ public Class<? extends Key<?>> getType(int field) {
+ return this.schema.get(field);
+ }
+
+ @Override
+ public Iterator<Entry<Integer, Class<? extends Key<?>>>> iterator() {
+ return this.schema.entrySet().iterator();
+ }
+
+ public int getNumTypes() {
+ return this.schema.size();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ return this.schema.hashCode() ^ getNumConnectionsThatContributed();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof SparseKeySchema) {
+ SparseKeySchema other = (SparseKeySchema) obj;
+ return this.schema.equals(other.schema) &&
+ this.getNumConnectionsThatContributed() == other.getNumConnectionsThatContributed();
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "<" + getNumConnectionsThatContributed() + "> : " + this.schema.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
new file mode 100644
index 0000000..bd35b5c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A traversal that collects cascading binary unions into a single n-ary
+ * union operator. The exception is, when on of the union inputs is materialized, such as in the
+ * static-code-path-cache in iterations.
+ */
+public class BinaryUnionReplacer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (this.seenBefore.add(visitable)) {
+ if (visitable instanceof IterationPlanNode) {
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+
+ if (visitable instanceof BinaryUnionPlanNode) {
+
+ final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ final Channel in1 = unionNode.getInput1();
+ final Channel in2 = unionNode.getInput2();
+
+ if (!unionNode.unionsStaticAndDynamicPath()) {
+
+ // both on static path, or both on dynamic path. we can collapse them
+ NAryUnionPlanNode newUnionNode;
+
+ List<Channel> inputs = new ArrayList<Channel>();
+ collect(in1, inputs);
+ collect(in2, inputs);
+
+ newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
+ unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
+
+ newUnionNode.setParallelism(unionNode.getParallelism());
+
+ for (Channel c : inputs) {
+ c.setTarget(newUnionNode);
+ }
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ else {
+ // union between the static and the dynamic path. we need to handle this for now
+ // through a special union operator
+
+ // make sure that the first input is the cached (static) and the second input is the dynamic
+ if (in1.isOnDynamicPath()) {
+ BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
+
+ in1.setTarget(newUnionNode);
+ in2.setTarget(newUnionNode);
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ }
+ }
+ }
+
+ public void collect(Channel in, List<Channel> inputs) {
+ if (in.getSource() instanceof NAryUnionPlanNode) {
+ // sanity check
+ if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
+ }
+ if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
+ }
+
+ inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
+ } else {
+ // is not a collapsed union node, so we take the channel directly
+ inputs.add(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
new file mode 100644
index 0000000..4730546
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * This traversal of the optimizer DAG computes the information needed to track
+ * branches and joins in the data flow. This is important to support plans
+ * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
+ * output into more than one other node).
+ */
+public final class BranchesVisitor implements Visitor<OptimizerNode> {
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ return node.getOpenBranches() == null;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode node) {
+ if (node instanceof IterationNode) {
+ ((IterationNode) node).acceptForStepFunction(this);
+ }
+
+ node.computeUnclosedBranchStack();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
new file mode 100644
index 0000000..160ef95
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.operators.GenericDataSinkBase;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Union;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
+import org.apache.flink.api.common.operators.base.CrossOperatorBase;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.api.common.operators.base.FilterOperatorBase;
+import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dag.BulkIterationNode;
+import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
+import org.apache.flink.optimizer.dag.CoGroupNode;
+import org.apache.flink.optimizer.dag.CollectorMapNode;
+import org.apache.flink.optimizer.dag.CrossNode;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dag.FilterNode;
+import org.apache.flink.optimizer.dag.FlatMapNode;
+import org.apache.flink.optimizer.dag.GroupCombineNode;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.JoinNode;
+import org.apache.flink.optimizer.dag.MapNode;
+import org.apache.flink.optimizer.dag.MapPartitionNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.PartitionNode;
+import org.apache.flink.optimizer.dag.ReduceNode;
+import org.apache.flink.optimizer.dag.SolutionSetNode;
+import org.apache.flink.optimizer.dag.SortPartitionNode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.optimizer.dag.WorksetNode;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This traversal creates the optimizer DAG from a program.
+ * It works as a visitor that walks the program's flow in a depth-first fashion, starting from the data sinks.
+ * During the descend, it creates an optimizer node for each operator, respectively data source or -sink.
+ * During the ascend, it connects the nodes to the full graph.
+ */
+public class GraphCreatingVisitor implements Visitor<Operator<?>> {
+
+ private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
+ // corresponding optimizer nodes
+
+ private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final int defaultParallelism; // the default degree of parallelism
+
+ private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
+
+ private final ExecutionMode defaultDataExchangeMode;
+
+ private final boolean forceDOP;
+
+
+ public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
+ this(null, false, defaultParallelism, defaultDataExchangeMode, null);
+ }
+
+ private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
+ ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
+ if (closure == null){
+ con2node = new HashMap<Operator<?>, OptimizerNode>();
+ } else {
+ con2node = closure;
+ }
+
+ this.sinks = new ArrayList<DataSinkNode>(2);
+ this.defaultParallelism = defaultParallelism;
+ this.parent = parent;
+ this.defaultDataExchangeMode = dataExchangeMode;
+ this.forceDOP = forceDOP;
+ }
+
+ public List<DataSinkNode> getSinks() {
+ return sinks;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public boolean preVisit(Operator<?> c) {
+ // check if we have been here before
+ if (this.con2node.containsKey(c)) {
+ return false;
+ }
+
+ final OptimizerNode n;
+
+ // create a node for the operator (or sink or source) if we have not been here before
+ if (c instanceof GenericDataSinkBase) {
+ DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
+ this.sinks.add(dsn);
+ n = dsn;
+ }
+ else if (c instanceof GenericDataSourceBase) {
+ n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
+ }
+ else if (c instanceof MapOperatorBase) {
+ n = new MapNode((MapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof MapPartitionOperatorBase) {
+ n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
+ n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FlatMapOperatorBase) {
+ n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FilterOperatorBase) {
+ n = new FilterNode((FilterOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof ReduceOperatorBase) {
+ n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof GroupCombineOperatorBase) {
+ n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof GroupReduceOperatorBase) {
+ n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof JoinOperatorBase) {
+ n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CoGroupOperatorBase) {
+ n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CrossOperatorBase) {
+ n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof BulkIterationBase) {
+ n = new BulkIterationNode((BulkIterationBase<?>) c);
+ }
+ else if (c instanceof DeltaIterationBase) {
+ n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
+ }
+ else if (c instanceof Union){
+ n = new BinaryUnionNode((Union<?>) c);
+ }
+ else if (c instanceof PartitionOperatorBase) {
+ n = new PartitionNode((PartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof SortPartitionOperatorBase) {
+ n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof BulkIterationBase.PartialSolutionPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final BulkIterationBase.PartialSolutionPlaceHolder<?> holder = (BulkIterationBase.PartialSolutionPlaceHolder<?>) c;
+ final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
+ final BulkIterationNode containingIterationNode =
+ (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof DeltaIterationBase.WorksetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final DeltaIterationBase.WorksetPlaceHolder<?> holder = (DeltaIterationBase.WorksetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ WorksetNode p = new WorksetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof DeltaIterationBase.SolutionSetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final DeltaIterationBase.SolutionSetPlaceHolder<?> holder = (DeltaIterationBase.SolutionSetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else {
+ throw new IllegalArgumentException("Unknown operator type: " + c);
+ }
+
+ this.con2node.put(c, n);
+
+ // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
+ // key-less reducer (all-reduce)
+ if (n.getParallelism() < 1) {
+ // set the degree of parallelism
+ int par = c.getDegreeOfParallelism();
+ if (par > 0) {
+ if (this.forceDOP && par != this.defaultParallelism) {
+ par = this.defaultParallelism;
+ Optimizer.LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
+ "currently fixed to the parallelism of the surrounding operator (the iteration).");
+ }
+ } else {
+ par = this.defaultParallelism;
+ }
+ n.setDegreeOfParallelism(par);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void postVisit(Operator<?> c) {
+
+ OptimizerNode n = this.con2node.get(c);
+
+ // first connect to the predecessors
+ n.setInput(this.con2node, this.defaultDataExchangeMode);
+ n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
+
+ // if the node represents a bulk iteration, we recursively translate the data flow now
+ if (n instanceof BulkIterationNode) {
+ final BulkIterationNode iterNode = (BulkIterationNode) n;
+ final BulkIterationBase<?> iter = iterNode.getIterationContract();
+
+ // pass a copy of the no iterative part into the iteration translation,
+ // in case the iteration references its closure
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
+ iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ BulkPartialSolutionNode partialSolution;
+
+ iter.getNextPartialSolution().accept(recursiveCreator);
+
+ partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
+ OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
+ if (partialSolution == null) {
+ throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
+ }
+
+
+ OptimizerNode terminationCriterion = null;
+
+ if (iter.getTerminationCriterion() != null) {
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+
+ // no intermediate node yet, traverse from the termination criterion to build the missing parts
+ if (terminationCriterion == null) {
+ iter.getTerminationCriterion().accept(recursiveCreator);
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+ }
+ }
+
+ iterNode.setPartialSolution(partialSolution);
+ iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(identifier);
+ }
+ else if (n instanceof WorksetIterationNode) {
+ final WorksetIterationNode iterNode = (WorksetIterationNode) n;
+ final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
+
+ // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
+ // One check is for free during the translation, we do the other check here as a pre-condition
+ {
+ StepFunctionValidator wsf = new StepFunctionValidator();
+ iter.getNextWorkset().accept(wsf);
+ if (!wsf.hasFoundWorkset()) {
+ throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
+ "This is a prerequisite in delta iterations.");
+ }
+ }
+
+ // calculate the closure of the anonymous function
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
+ this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ // descend from the solution set delta. check that it depends on both the workset
+ // and the solution set. If it does depend on both, this descend should create both nodes
+ iter.getSolutionSetDelta().accept(recursiveCreator);
+
+ final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
+
+ if (worksetNode == null) {
+ throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
+ "This is a prerequisite in delta iterations.");
+ }
+
+ iter.getNextWorkset().accept(recursiveCreator);
+
+ SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
+
+ if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
+ solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
+ }
+ else {
+ for (DagConnection conn : solutionSetNode.getOutgoingConnections()) {
+ OptimizerNode successor = conn.getTarget();
+
+ if (successor.getClass() == JoinNode.class) {
+ // find out which input to the match the solution set is
+ JoinNode mn = (JoinNode) successor;
+ if (mn.getFirstPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(0);
+ } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else if (successor.getClass() == CoGroupNode.class) {
+ CoGroupNode cg = (CoGroupNode) successor;
+ if (cg.getFirstPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(0);
+ } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else {
+ throw new InvalidProgramException(
+ "Error: The only operations allowed on the solution set are Join and CoGroup.");
+ }
+ }
+ }
+
+ final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
+ final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
+
+ // set the step function nodes to the iteration node
+ iterNode.setPartialSolution(solutionSetNode, worksetNode);
+ iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(pathIdentifier);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
new file mode 100644
index 0000000..b5c09e5
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * This traversal of the optimizer DAG assigns IDs to each node (in a pre-order fashion),
+ * and calls each node to compute its estimates. The latter happens in the postVisit function,
+ * where it is guaranteed that all predecessors have computed their estimates.
+ */
+public class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
+
+ private final DataStatistics statistics;
+
+ private int id = 1;
+
+ public IdAndEstimatesVisitor(DataStatistics statistics) {
+ this.statistics = statistics;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return visitable.getId() == -1;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ // the node ids
+ visitable.initId(this.id++);
+
+ // connections need to figure out their maximum path depths
+ for (DagConnection conn : visitable.getIncomingConnections()) {
+ conn.initMaxDepth();
+ }
+ for (DagConnection conn : visitable.getBroadcastConnections()) {
+ conn.initMaxDepth();
+ }
+
+ // the estimates
+ visitable.computeOutputEstimates(this.statistics);
+
+ // if required, recurse into the step function
+ if (visitable instanceof IterationNode) {
+ ((IterationNode) visitable).acceptForStepFunction(this);
+ }
+ }
+}
[25/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
deleted file mode 100644
index f6885c5..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReduceAllTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.junit.Test;
-
-/**
- * This test case has been created to validate a bug that occurred when
- * the ReduceOperator was used without a grouping key.
- */
-@SuppressWarnings({"serial", "deprecation"})
-public class ReduceAllTest extends CompilerTestBase {
-
- @Test
- public void testReduce() {
- // construct the plan
- FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
- ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce()).name("Reduce1").input(source).build();
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
- sink.setInput(reduce1);
- Plan plan = new Plan(sink, "AllReduce Test");
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
-
-
- try {
- OptimizedPlan oPlan = compileNoStats(plan);
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
deleted file mode 100644
index da44b59..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/ReplicatingDataSourceTest.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.MapPartitionFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.io.ReplicatingInputFormat;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.CsvInputFormat;
-import org.apache.flink.api.java.operators.DataSink;
-import org.apache.flink.api.java.operators.translation.JavaPlan;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.core.fs.FileInputSplit;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Collector;
-import org.junit.Assert;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "deprecation"})
-public class ReplicatingDataSourceTest extends CompilerTestBase {
-
- /**
- * Tests join program with replicated data source.
- */
- @Test
- public void checkJoinWithReplicatedSourceInput() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests join program with replicated data source behind map.
- */
- @Test
- public void checkJoinWithReplicatedSourceInputBehindMap() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .map(new IdMap())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests join program with replicated data source behind filter.
- */
- @Test
- public void checkJoinWithReplicatedSourceInputBehindFilter() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .filter(new NoFilter())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests join program with replicated data source behind flatMap.
- */
- @Test
- public void checkJoinWithReplicatedSourceInputBehindFlatMap() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .flatMap(new IdFlatMap())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests join program with replicated data source behind map partition.
- */
- @Test
- public void checkJoinWithReplicatedSourceInputBehindMapPartition() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .mapPartition(new IdPMap())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests join program with replicated data source behind multiple map ops.
- */
- @Test
- public void checkJoinWithReplicatedSourceInputBehindMultiMaps() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .filter(new NoFilter())
- .mapPartition(new IdPMap())
- .flatMap(new IdFlatMap())
- .map(new IdMap())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when join should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType joinIn1 = joinNode.getInput1().getShipStrategy();
- ShipStrategyType joinIn2 = joinNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, joinIn2);
- }
-
- /**
- * Tests cross program with replicated data source.
- */
- @Test
- public void checkCrossWithReplicatedSourceInput() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .cross(source2)
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when cross should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
- ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
- }
-
- /**
- * Tests cross program with replicated data source behind map and filter.
- */
- @Test
- public void checkCrossWithReplicatedSourceInputBehindMap() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .map(new IdMap())
- .filter(new NoFilter())
- .cross(source2)
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- // when cross should have forward strategy on both sides
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode crossNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- ShipStrategyType crossIn1 = crossNode.getInput1().getShipStrategy();
- ShipStrategyType crossIn2 = crossNode.getInput2().getShipStrategy();
-
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn1);
- Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, crossIn2);
- }
-
- /**
- * Tests compiler fail for join program with replicated data source and changing DOP.
- */
- @Test(expected = CompilerException.class)
- public void checkJoinWithReplicatedSourceInputChangingDOP() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .join(source2).where("*").equalTo("*").setParallelism(DEFAULT_PARALLELISM+2)
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
- }
-
- /**
- * Tests compiler fail for join program with replicated data source behind map and changing DOP.
- */
- @Test(expected = CompilerException.class)
- public void checkJoinWithReplicatedSourceInputBehindMapChangingDOP() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .map(new IdMap()).setParallelism(DEFAULT_PARALLELISM+1)
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- }
-
- /**
- * Tests compiler fail for join program with replicated data source behind reduce.
- */
- @Test(expected = CompilerException.class)
- public void checkJoinWithReplicatedSourceInputBehindReduce() {
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .reduce(new LastReduce())
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
- }
-
- /**
- * Tests compiler fail for join program with replicated data source behind rebalance.
- */
- @Test(expected = CompilerException.class)
- public void checkJoinWithReplicatedSourceInputBehindRebalance() {
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif =
- new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new CsvInputFormat<Tuple1<String>>(new Path("/some/path"), String.class));
-
- DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO));
- DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class);
-
- DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1
- .rebalance()
- .join(source2).where("*").equalTo("*")
- .writeAsText("/some/newpath");
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
- }
-
-
- public static class IdMap<T> implements MapFunction<T,T> {
-
- @Override
- public T map(T value) throws Exception {
- return value;
- }
- }
-
- public static class NoFilter<T> implements FilterFunction<T> {
-
- @Override
- public boolean filter(T value) throws Exception {
- return false;
- }
- }
-
- public static class IdFlatMap<T> implements FlatMapFunction<T,T> {
-
- @Override
- public void flatMap(T value, Collector<T> out) throws Exception {
- out.collect(value);
- }
- }
-
- public static class IdPMap<T> implements MapPartitionFunction<T,T> {
-
- @Override
- public void mapPartition(Iterable<T> values, Collector<T> out) throws Exception {
- for(T v : values) {
- out.collect(v);
- }
- }
- }
-
- public static class LastReduce<T> implements ReduceFunction<T> {
-
- @Override
- public T reduce(T value1, T value2) throws Exception {
- return value2;
- }
- }
-
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
deleted file mode 100644
index 1fe16bb..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/SemanticPropertiesAPIToPlanTest.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.translation.JavaPlan;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-import org.junit.Assert;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class SemanticPropertiesAPIToPlanTest extends CompilerTestBase {
-
- @Test
- public void forwardFieldsTestMapReduce() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- set = set.map(new MockMapper()).withForwardedFields("*")
- .groupBy(0)
- .reduce(new MockReducer()).withForwardedFields("f0->f1")
- .map(new MockMapper()).withForwardedFields("*")
- .groupBy(1)
- .reduce(new MockReducer()).withForwardedFields("*");
-
- set.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- oPlan.accept(new Visitor<PlanNode>() {
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperatorBase) {
- for (Channel input: visitable.getInputs()) {
- GlobalProperties gprops = visitable.getGlobalProperties();
- LocalProperties lprops = visitable.getLocalProperties();
-
- Assert.assertTrue("Reduce should just forward the input if it is already partitioned",
- input.getShipStrategy() == ShipStrategyType.FORWARD);
- Assert.assertTrue("Wrong GlobalProperties on Reducer",
- gprops.isPartitionedOnFields(new FieldSet(1)));
- Assert.assertTrue("Wrong GlobalProperties on Reducer",
- gprops.getPartitioning() == PartitioningProperty.HASH_PARTITIONED);
- Assert.assertTrue("Wrong LocalProperties on Reducer",
- lprops.getGroupedFields().contains(1));
- }
- }
- if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof MapOperatorBase) {
- for (Channel input: visitable.getInputs()) {
- GlobalProperties gprops = visitable.getGlobalProperties();
- LocalProperties lprops = visitable.getLocalProperties();
-
- Assert.assertTrue("Map should just forward the input if it is already partitioned",
- input.getShipStrategy() == ShipStrategyType.FORWARD);
- Assert.assertTrue("Wrong GlobalProperties on Mapper",
- gprops.isPartitionedOnFields(new FieldSet(1)));
- Assert.assertTrue("Wrong GlobalProperties on Mapper",
- gprops.getPartitioning() == PartitioningProperty.HASH_PARTITIONED);
- Assert.assertTrue("Wrong LocalProperties on Mapper",
- lprops.getGroupedFields().contains(1));
- }
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
-
- }
- });
- }
-
- @Test
- public void forwardFieldsTestJoin() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> in1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> in2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- in1 = in1.map(new MockMapper()).withForwardedFields("*")
- .groupBy(0)
- .reduce(new MockReducer()).withForwardedFields("f0->f1");
- in2 = in2.map(new MockMapper()).withForwardedFields("*")
- .groupBy(1)
- .reduce(new MockReducer()).withForwardedFields("f1->f2");
- DataSet<Tuple3<Integer, Integer, Integer>> out = in1.join(in2).where(1).equalTo(2).with(new MockJoin());
-
- out.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- oPlan.accept(new Visitor<PlanNode>() {
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof DualInputPlanNode && visitable.getProgramOperator() instanceof JoinOperatorBase) {
- DualInputPlanNode node = ((DualInputPlanNode) visitable);
-
- final Channel inConn1 = node.getInput1();
- final Channel inConn2 = node.getInput2();
-
- Assert.assertTrue("Join should just forward the input if it is already partitioned",
- inConn1.getShipStrategy() == ShipStrategyType.FORWARD);
- Assert.assertTrue("Join should just forward the input if it is already partitioned",
- inConn2.getShipStrategy() == ShipStrategyType.FORWARD);
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
-
- }
- });
- }
-
- public static class MockMapper implements MapFunction<Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
- @Override
- public Tuple3<Integer, Integer, Integer> map(Tuple3<Integer, Integer, Integer> value) throws Exception {
- return null;
- }
- }
-
- public static class MockReducer implements ReduceFunction<Tuple3<Integer, Integer, Integer>> {
- @Override
- public Tuple3<Integer, Integer, Integer> reduce(Tuple3<Integer, Integer, Integer> value1, Tuple3<Integer, Integer, Integer> value2) throws Exception {
- return null;
- }
- }
-
- public static class MockJoin implements JoinFunction<Tuple3<Integer, Integer, Integer>,
- Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
-
- @Override
- public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
- return null;
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
deleted file mode 100644
index 40b54e0..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/SortPartialReuseTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-
-@SuppressWarnings("serial")
-public class SortPartialReuseTest extends CompilerTestBase {
-
- @Test
- public void testPartialPartitioningReuse() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input
- .partitionByHash(0)
- .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
-
- .groupBy(0, 1)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
-
- .groupBy(0)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
-
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer2 = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode reducer1 = (SingleInputPlanNode) reducer2.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
-
- // should be locally forwarding, reusing sort and partitioning
- assertEquals(ShipStrategyType.FORWARD, reducer2.getInput().getShipStrategy());
- assertEquals(LocalStrategy.NONE, reducer2.getInput().getLocalStrategy());
-
- assertEquals(ShipStrategyType.FORWARD, reducer1.getInput().getShipStrategy());
- assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningNotReused() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input
- .partitionCustom(new Partitioner<Long>() {
- @Override
- public int partition(Long key, int numPartitions) { return 0; }
- }, 0)
- .map(new IdentityMapper<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
-
- .groupBy(0, 1)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>()).withForwardedFields("0", "1", "2")
-
- .groupBy(1)
- .reduceGroup(new IdentityGroupReducer<Tuple3<Long,Long,Long>>())
-
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode reducer2 = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode combiner = (SingleInputPlanNode) reducer2.getInput().getSource();
- SingleInputPlanNode reducer1 = (SingleInputPlanNode) combiner.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
-
- // should be locally forwarding, reusing sort and partitioning
- assertEquals(ShipStrategyType.PARTITION_HASH, reducer2.getInput().getShipStrategy());
- assertEquals(LocalStrategy.COMBININGSORT, reducer2.getInput().getLocalStrategy());
-
- assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
- assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy());
-
- assertEquals(ShipStrategyType.FORWARD, reducer1.getInput().getShipStrategy());
- assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
deleted file mode 100644
index 92b4fc5..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionBetweenDynamicAndStaticPathTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class UnionBetweenDynamicAndStaticPathTest extends CompilerTestBase {
-
- @Test
- public void testUnionStaticFirst() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> input1 = env.generateSequence(1, 10);
- DataSet<Long> input2 = env.generateSequence(1, 10);
-
- IterativeDataSet<Long> iteration = input1.iterate(10);
-
- DataSet<Long> result = iteration.closeWith(
- input2.union(input2).union(iteration.union(iteration)));
-
- result.print();
- result.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(2, op.getDataSinks().size());
-
- BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
- BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
- NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
- NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
-
- assertTrue(mixedUnion.unionsStaticAndDynamicPath());
- assertFalse(mixedUnion.getInput1().isOnDynamicPath());
- assertTrue(mixedUnion.getInput2().isOnDynamicPath());
- assertTrue(mixedUnion.getInput1().getTempMode().isCached());
-
- for (Channel c : staticUnion.getInputs()) {
- assertFalse(c.isOnDynamicPath());
- }
- for (Channel c : dynamicUnion.getInputs()) {
- assertTrue(c.isOnDynamicPath());
- }
-
- assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
- assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
- assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testUnionStaticSecond() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> input1 = env.generateSequence(1, 10);
- DataSet<Long> input2 = env.generateSequence(1, 10);
-
- IterativeDataSet<Long> iteration = input1.iterate(10);
-
- DataSet<Long> iterResult = iteration
- .closeWith(iteration.union(iteration).union(input2.union(input2)));
-
- iterResult.print();
- iterResult.print();
-
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- assertEquals(2, op.getDataSinks().size());
-
- BulkIterationPlanNode iterPlan = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
-
- SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction();
- BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource();
- NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource();
- NAryUnionPlanNode dynamicUnion = (NAryUnionPlanNode) mixedUnion.getInput2().getSource();
-
- assertTrue(mixedUnion.unionsStaticAndDynamicPath());
- assertFalse(mixedUnion.getInput1().isOnDynamicPath());
- assertTrue(mixedUnion.getInput2().isOnDynamicPath());
- assertTrue(mixedUnion.getInput1().getTempMode().isCached());
-
- assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0);
- assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0);
- assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
-
- for (Channel c : staticUnion.getInputs()) {
- assertFalse(c.isOnDynamicPath());
- }
- for (Channel c : dynamicUnion.getInputs()) {
- assertTrue(c.isOnDynamicPath());
- }
-
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
deleted file mode 100644
index 5d15ed8..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionPropertyPropagationTest.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import java.util.Iterator;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.Visitor;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityReduce;
-
-
-@SuppressWarnings({"serial", "deprecation"})
-public class UnionPropertyPropagationTest extends CompilerTestBase {
-
- @SuppressWarnings("unchecked")
- @Test
- public void testUnionPropertyOldApiPropagation() {
- // construct the plan
-
- FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), IN_FILE);
- FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), IN_FILE);
-
- ReduceOperator redA = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(sourceA)
- .build();
- ReduceOperator redB = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
- .input(sourceB)
- .build();
-
- ReduceOperator globalRed = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).build();
- globalRed.addInput(redA);
- globalRed.addInput(redB);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, globalRed);
-
- // return the plan
- Plan plan = new Plan(sink, "Union Property Propagation");
-
- OptimizedPlan oPlan = compileNoStats(plan);
-
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- // Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
-
- oPlan.accept(new Visitor<PlanNode>() {
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperator) {
- for (Channel inConn : visitable.getInputs()) {
- Assert.assertTrue("Reduce should just forward the input if it is already partitioned",
- inConn.getShipStrategy() == ShipStrategyType.FORWARD);
- }
- //just check latest ReduceNode
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
- // DO NOTHING
- }
- });
- }
-
- @Test
- public void testUnionNewApiAssembly() {
- final int NUM_INPUTS = 4;
-
- // construct the plan it will be multiple flat maps, all unioned
- // and the "unioned" dataSet will be grouped
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<String> source = env.readTextFile(IN_FILE);
- DataSet<Tuple2<String, Integer>> lastUnion = source.flatMap(new DummyFlatMap());
-
- for (int i = 1; i< NUM_INPUTS; i++){
- lastUnion = lastUnion.union(source.flatMap(new DummyFlatMap()));
- }
-
- DataSet<Tuple2<String, Integer>> result = lastUnion.groupBy(0).aggregate(Aggregations.SUM, 1);
- result.writeAsText(OUT_FILE);
-
- // return the plan
- Plan plan = env.createProgramPlan("Test union on new java-api");
- OptimizedPlan oPlan = compileNoStats(plan);
- JobGraphGenerator jobGen = new JobGraphGenerator();
-
- // Compile plan to verify that no error is thrown
- jobGen.compileJobGraph(oPlan);
-
- oPlan.accept(new Visitor<PlanNode>() {
-
- @Override
- public boolean preVisit(PlanNode visitable) {
-
- /* Test on the union output connections
- * It must be under the GroupOperator and the strategy should be forward
- */
- if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof GroupReduceOperatorBase){
- final Channel inConn = ((SingleInputPlanNode) visitable).getInput();
- Assert.assertTrue("Union should just forward the Partitioning",
- inConn.getShipStrategy() == ShipStrategyType.FORWARD );
- Assert.assertTrue("Union Node should be under Group operator",
- inConn.getSource() instanceof NAryUnionPlanNode );
- }
-
- /* Test on the union input connections
- * Must be NUM_INPUTS input connections, all FlatMapOperators with a own partitioning strategy(propably hash)
- */
- if (visitable instanceof NAryUnionPlanNode) {
- int numberInputs = 0;
- for (Iterator<Channel> inputs = visitable.getInputs().iterator(); inputs.hasNext(); numberInputs++) {
- final Channel inConn = inputs.next();
- PlanNode inNode = inConn.getSource();
- Assert.assertTrue("Input of Union should be FlatMapOperators",
- inNode.getProgramOperator() instanceof FlatMapOperatorBase);
- Assert.assertTrue("Shipment strategy under union should partition the data",
- inConn.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
- }
-
- Assert.assertTrue("NAryUnion should have " + NUM_INPUTS + " inputs", numberInputs == NUM_INPUTS);
- return false;
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
- // DO NOTHING
- }
- });
- }
-
- public static final class DummyFlatMap extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- out.collect(new Tuple2<String, Integer>(value, 0));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
deleted file mode 100644
index 1e4124c..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/UnionReplacementTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.junit.Test;
-
-import static org.junit.Assert.fail;
-
-@SuppressWarnings("serial")
-public class UnionReplacementTest extends CompilerTestBase {
-
- @Test
- public void testUnionReplacement() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<String> input1 = env.fromElements("test1");
- DataSet<String> input2 = env.fromElements("test2");
-
- DataSet<String> union = input1.union(input2);
-
- union.print();
- union.print();
-
- Plan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileNoStats(plan);
- JobGraphGenerator jobGen = new JobGraphGenerator();
- jobGen.compileJobGraph(oPlan);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
deleted file mode 100644
index 80c0bda..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationCornerCasesTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class WorksetIterationCornerCasesTest extends CompilerTestBase {
-
- @Test
- public void testWorksetIterationNotDependingOnSolutionSet() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 100).map(new Duplicator<Long>());
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 100, 1);
-
- DataSet<Tuple2<Long, Long>> iterEnd = iteration.getWorkset().map(new TestMapper<Tuple2<Long,Long>>());
- iteration.closeWith(iterEnd, iterEnd).print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
- assertTrue(wipn.getSolutionSetPlanNode().getOutgoingChannels().isEmpty());
-
- JobGraphGenerator jgg = new JobGraphGenerator();
- jgg.compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- private static final class Duplicator<T> implements MapFunction<T, Tuple2<T, T>> {
- @Override
- public Tuple2<T, T> map(T value) {
- return new Tuple2<T, T>(value, value);
- }
- }
-
- private static final class TestMapper<T> implements MapFunction<T, T> {
- @Override
- public T map(T value) {
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
deleted file mode 100644
index 6e7c0a3..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/WorksetIterationsRecordApiCompilerTest.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.record.operators.DeltaIteration;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.JoinOperator;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.DummyInputFormat;
-import org.apache.flink.optimizer.util.DummyMatchStub;
-import org.apache.flink.optimizer.util.DummyNonPreservingMatchStub;
-import org.apache.flink.optimizer.util.DummyOutputFormat;
-import org.apache.flink.optimizer.util.IdentityMap;
-import org.apache.flink.optimizer.util.IdentityReduce;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.types.LongValue;
-import org.junit.Test;
-
-
-/**
-* Tests that validate optimizer choices when using operators that are requesting certain specific execution
-* strategies.
-*/
-@SuppressWarnings("deprecation")
-public class WorksetIterationsRecordApiCompilerTest extends CompilerTestBase {
-
- private static final long serialVersionUID = 1L;
-
- private static final String ITERATION_NAME = "Test Workset Iteration";
- private static final String JOIN_WITH_INVARIANT_NAME = "Test Join Invariant";
- private static final String JOIN_WITH_SOLUTION_SET = "Test Join SolutionSet";
- private static final String NEXT_WORKSET_REDUCER_NAME = "Test Reduce Workset";
- private static final String SOLUTION_DELTA_MAPPER_NAME = "Test Map Delta";
-
- private final FieldList list0 = new FieldList(0);
-
- @Test
- public void testRecordApiWithDeferredSoltionSetUpdateWithMapper() {
- Plan plan = getRecordTestPlan(false, true);
-
- OptimizedPlan oPlan;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- return; // silence the compiler
- }
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
- SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
-
- // verify reducer
- assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
- assertEquals(list0, worksetReducer.getKeys(0));
-
- // currently, the system may partition before or after the mapper
- ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
- ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
-
- assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
- (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
-
- @Test
- public void testRecordApiWithDeferredSoltionSetUpdateWithNonPreservingJoin() {
- Plan plan = getRecordTestPlan(false, false);
-
- OptimizedPlan oPlan;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- return; // silence the compiler
- }
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
-
- // verify reducer
- assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
- assertEquals(list0, worksetReducer.getKeys(0));
-
-
- // verify solution delta
- assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
-
- @Test
- public void testRecordApiWithDirectSoltionSetUpdate() {
- Plan plan = getRecordTestPlan(true, false);
-
- OptimizedPlan oPlan;
- try {
- oPlan = compileNoStats(plan);
- } catch(CompilerException ce) {
- ce.printStackTrace();
- fail("The pact compiler is unable to compile this plan correctly.");
- return; // silence the compiler
- }
-
- OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
- DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
- DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
- SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
-
- // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
- // the in-loop partitioning is before the final reducer
-
- // verify joinWithInvariant
- assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
- assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
-
- // verify joinWithSolutionSet
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
-
- // verify reducer
- assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
- assertEquals(list0, worksetReducer.getKeys(0));
-
-
- // verify solution delta
- assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
- assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
-
- new JobGraphGenerator().compileJobGraph(oPlan);
- }
-
- private Plan getRecordTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
- FileDataSource solutionSetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Solution Set");
- FileDataSource worksetInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Workset");
-
- FileDataSource invariantInput = new FileDataSource(new DummyInputFormat(), IN_FILE, "Invariant Input");
-
- DeltaIteration iteration = new DeltaIteration(0, ITERATION_NAME);
- iteration.setInitialSolutionSet(solutionSetInput);
- iteration.setInitialWorkset(worksetInput);
- iteration.setMaximumNumberOfIterations(100);
-
- JoinOperator joinWithInvariant = JoinOperator.builder(new DummyMatchStub(), LongValue.class, 0, 0)
- .input1(iteration.getWorkset())
- .input2(invariantInput)
- .name(JOIN_WITH_INVARIANT_NAME)
- .build();
-
- JoinOperator joinWithSolutionSet = JoinOperator.builder(
- joinPreservesSolutionSet ? new DummyMatchStub() : new DummyNonPreservingMatchStub(), LongValue.class, 0, 0)
- .input1(iteration.getSolutionSet())
- .input2(joinWithInvariant)
- .name(JOIN_WITH_SOLUTION_SET)
- .build();
-
- ReduceOperator nextWorkset = ReduceOperator.builder(new IdentityReduce(), LongValue.class, 0)
- .input(joinWithSolutionSet)
- .name(NEXT_WORKSET_REDUCER_NAME)
- .build();
-
- if (mapBeforeSolutionDelta) {
- MapOperator mapper = MapOperator.builder(new IdentityMap())
- .input(joinWithSolutionSet)
- .name(SOLUTION_DELTA_MAPPER_NAME)
- .build();
- iteration.setSolutionSetDelta(mapper);
- } else {
- iteration.setSolutionSetDelta(joinWithSolutionSet);
- }
-
- iteration.setNextWorkset(nextWorkset);
-
- FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, iteration, "Sink");
-
- Plan plan = new Plan(sink);
- plan.setDefaultParallelism(DEFAULT_PARALLELISM);
- return plan;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
deleted file mode 100644
index af03eec..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/costs/DefaultCostEstimatorTest.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.costs;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.optimizer.dag.EstimateProvider;
-import org.junit.Test;
-
-/**
- * Tests for the cost formulas in the {@link DefaultCostEstimator}. Most of the tests establish relative
- * relationships.
- */
-public class DefaultCostEstimatorTest {
-
- // estimates
-
- private static final long SMALL_DATA_SIZE = 10000;
- private static final long SMALL_RECORD_COUNT = 100;
-
- private static final long MEDIUM_DATA_SIZE = 500000000L;
- private static final long MEDIUM_RECORD_COUNT = 500000L;
-
- private static final long BIG_DATA_SIZE = 100000000000L;
- private static final long BIG_RECORD_COUNT = 100000000L;
-
- private static final EstimateProvider UNKNOWN_ESTIMATES = new UnknownEstimates();
- private static final EstimateProvider ZERO_ESTIMATES = new Estimates(0, 0);
- private static final EstimateProvider SMALL_ESTIMATES = new Estimates(SMALL_DATA_SIZE, SMALL_RECORD_COUNT);
- private static final EstimateProvider MEDIUM_ESTIMATES = new Estimates(MEDIUM_DATA_SIZE, MEDIUM_RECORD_COUNT);
- private static final EstimateProvider BIG_ESTIMATES = new Estimates(BIG_DATA_SIZE, BIG_RECORD_COUNT);
-
- private final CostEstimator costEstimator = new DefaultCostEstimator();
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testShipStrategiesIsolated() {
- testShipStrategiesIsolated(UNKNOWN_ESTIMATES, 1);
- testShipStrategiesIsolated(UNKNOWN_ESTIMATES, 10);
- testShipStrategiesIsolated(ZERO_ESTIMATES, 1);
- testShipStrategiesIsolated(ZERO_ESTIMATES, 10);
- testShipStrategiesIsolated(SMALL_ESTIMATES, 1);
- testShipStrategiesIsolated(SMALL_ESTIMATES, 10);
- testShipStrategiesIsolated(BIG_ESTIMATES, 1);
- testShipStrategiesIsolated(BIG_ESTIMATES, 10);
- }
-
- private void testShipStrategiesIsolated(EstimateProvider estimates, int targetParallelism) {
- Costs random = new Costs();
- costEstimator.addRandomPartitioningCost(estimates, random);
-
- Costs hash = new Costs();
- costEstimator.addHashPartitioningCost(estimates, hash);
-
- Costs range = new Costs();
- costEstimator.addRangePartitionCost(estimates, range);
-
- Costs broadcast = new Costs();
- costEstimator.addBroadcastCost(estimates, targetParallelism, broadcast);
-
- int randomVsHash = random.compareTo(hash);
- int hashVsRange = hash.compareTo(range);
- int hashVsBroadcast = hash.compareTo(broadcast);
- int rangeVsBroadcast = range.compareTo(broadcast);
-
- // repartition random is at most as expensive as hash partitioning
- assertTrue(randomVsHash <= 0);
-
- // range partitioning is always more expensive than hash partitioning
- assertTrue(hashVsRange < 0);
-
- // broadcasting is always more expensive than hash partitioning
- if (targetParallelism > 1) {
- assertTrue(hashVsBroadcast < 0);
- } else {
- assertTrue(hashVsBroadcast <= 0);
- }
-
- // range partitioning is not more expensive than broadcasting
- if (targetParallelism > 1) {
- assertTrue(rangeVsBroadcast < 0);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testShipStrategyCombinationsPlain() {
- Costs hashBothSmall = new Costs();
- Costs hashSmallAndLarge = new Costs();
- Costs hashBothLarge = new Costs();
-
- Costs hashSmallBcLarge10 = new Costs();
- Costs hashLargeBcSmall10 = new Costs();
-
- Costs hashSmallBcLarge1000 = new Costs();
- Costs hashLargeBcSmall1000 = new Costs();
-
- Costs forwardSmallBcLarge10 = new Costs();
- Costs forwardLargeBcSmall10 = new Costs();
-
- Costs forwardSmallBcLarge1000 = new Costs();
- Costs forwardLargeBcSmall1000 = new Costs();
-
- costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashBothSmall);
- costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashBothSmall);
-
- costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallAndLarge);
- costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashSmallAndLarge);
-
- costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashBothLarge);
- costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashBothLarge);
-
- costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallBcLarge10);
- costEstimator.addBroadcastCost(BIG_ESTIMATES, 10, hashSmallBcLarge10);
-
- costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashLargeBcSmall10);
- costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 10, hashLargeBcSmall10);
-
- costEstimator.addHashPartitioningCost(MEDIUM_ESTIMATES, hashSmallBcLarge1000);
- costEstimator.addBroadcastCost(BIG_ESTIMATES, 1000, hashSmallBcLarge1000);
-
- costEstimator.addHashPartitioningCost(BIG_ESTIMATES, hashLargeBcSmall1000);
- costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 1000, hashLargeBcSmall1000);
-
- costEstimator.addBroadcastCost(BIG_ESTIMATES, 10, forwardSmallBcLarge10);
-
- costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 10, forwardLargeBcSmall10);
-
- costEstimator.addBroadcastCost(BIG_ESTIMATES, 1000, forwardSmallBcLarge1000);
-
- costEstimator.addBroadcastCost(MEDIUM_ESTIMATES, 1000, forwardLargeBcSmall1000);
-
- // hash cost is roughly monotonous
- assertTrue(hashBothSmall.compareTo(hashSmallAndLarge) < 0);
- assertTrue(hashSmallAndLarge.compareTo(hashBothLarge) < 0);
-
- // broadcast the smaller is better
- assertTrue(hashLargeBcSmall10.compareTo(hashSmallBcLarge10) < 0);
- assertTrue(forwardLargeBcSmall10.compareTo(forwardSmallBcLarge10) < 0);
- assertTrue(hashLargeBcSmall1000.compareTo(hashSmallBcLarge1000) < 0);
- assertTrue(forwardLargeBcSmall1000.compareTo(forwardSmallBcLarge1000) < 0);
-
- // broadcasting small and forwarding large is better than partition both, given size difference
- assertTrue(forwardLargeBcSmall10.compareTo(hashSmallAndLarge) < 0);
-
- // broadcasting too far is expensive again
- assertTrue(forwardLargeBcSmall1000.compareTo(hashSmallAndLarge) > 0);
-
- // assert weight is respected
- assertTrue(hashSmallBcLarge10.compareTo(hashSmallBcLarge1000) < 0);
- assertTrue(hashLargeBcSmall10.compareTo(hashLargeBcSmall1000) < 0);
- assertTrue(forwardSmallBcLarge10.compareTo(forwardSmallBcLarge1000) < 0);
- assertTrue(forwardLargeBcSmall10.compareTo(forwardLargeBcSmall1000) < 0);
-
- // forward versus hash
- assertTrue(forwardSmallBcLarge10.compareTo(hashSmallBcLarge10) < 0);
- assertTrue(forwardSmallBcLarge1000.compareTo(hashSmallBcLarge1000) < 0);
- assertTrue(forwardLargeBcSmall10.compareTo(hashLargeBcSmall10) < 0);
- assertTrue(forwardLargeBcSmall1000.compareTo(hashLargeBcSmall1000) < 0);
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testShipStrategyCombinationsWithUnknowns() {
- testShipStrategyCombinationsWithUnknowns(UNKNOWN_ESTIMATES);
- testShipStrategyCombinationsWithUnknowns(ZERO_ESTIMATES);
- testShipStrategyCombinationsWithUnknowns(SMALL_ESTIMATES);
- testShipStrategyCombinationsWithUnknowns(MEDIUM_ESTIMATES);
- testShipStrategyCombinationsWithUnknowns(BIG_ESTIMATES);
- }
-
- private void testShipStrategyCombinationsWithUnknowns(EstimateProvider knownEstimates) {
- Costs hashBoth = new Costs();
- Costs bcKnown10 = new Costs();
- Costs bcUnknown10 = new Costs();
- Costs bcKnown1000 = new Costs();
- Costs bcUnknown1000 = new Costs();
-
- costEstimator.addHashPartitioningCost(knownEstimates, hashBoth);
- costEstimator.addHashPartitioningCost(UNKNOWN_ESTIMATES, hashBoth);
-
- costEstimator.addBroadcastCost(knownEstimates, 10, bcKnown10);
-
- costEstimator.addBroadcastCost(UNKNOWN_ESTIMATES, 10, bcUnknown10);
-
- costEstimator.addBroadcastCost(knownEstimates, 1000, bcKnown1000);
-
- costEstimator.addBroadcastCost(UNKNOWN_ESTIMATES, 1000, bcUnknown1000);
-
- // if we do not know one of them, hashing both should be cheaper than anything
- assertTrue(hashBoth.compareTo(bcKnown10) < 0);
- assertTrue(hashBoth.compareTo(bcUnknown10) < 0);
- assertTrue(hashBoth.compareTo(bcKnown1000) < 0);
- assertTrue(hashBoth.compareTo(bcUnknown1000) < 0);
-
- // there should be no bias in broadcasting a known or unknown size input
- assertTrue(bcKnown10.compareTo(bcUnknown10) == 0);
- assertTrue(bcKnown1000.compareTo(bcUnknown1000) == 0);
-
- // replication factor does matter
- assertTrue(bcKnown10.compareTo(bcKnown1000) < 0);
- assertTrue(bcUnknown10.compareTo(bcUnknown1000) < 0);
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testJoinCostFormulasPlain() {
-
- // hash join costs
-
- Costs hashBothSmall = new Costs();
- Costs hashBothLarge = new Costs();
- Costs hashSmallBuild = new Costs();
- Costs hashLargeBuild = new Costs();
-
- costEstimator.addHybridHashCosts(SMALL_ESTIMATES, BIG_ESTIMATES, hashSmallBuild, 1);
- costEstimator.addHybridHashCosts(BIG_ESTIMATES, SMALL_ESTIMATES, hashLargeBuild, 1);
- costEstimator.addHybridHashCosts(SMALL_ESTIMATES, SMALL_ESTIMATES, hashBothSmall, 1);
- costEstimator.addHybridHashCosts(BIG_ESTIMATES, BIG_ESTIMATES, hashBothLarge, 1);
-
- assertTrue(hashBothSmall.compareTo(hashSmallBuild) < 0);
- assertTrue(hashSmallBuild.compareTo(hashLargeBuild) < 0);
- assertTrue(hashLargeBuild.compareTo(hashBothLarge) < 0);
-
- // merge join costs
-
- Costs mergeBothSmall = new Costs();
- Costs mergeBothLarge = new Costs();
- Costs mergeSmallFirst = new Costs();
- Costs mergeSmallSecond = new Costs();
-
- costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallFirst);
- costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallFirst);
- costEstimator.addLocalMergeCost(SMALL_ESTIMATES, BIG_ESTIMATES, mergeSmallFirst, 1);
-
- costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallSecond);
- costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallSecond);
- costEstimator.addLocalMergeCost(BIG_ESTIMATES, SMALL_ESTIMATES, mergeSmallSecond, 1);
-
- costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
- costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
- costEstimator.addLocalMergeCost(SMALL_ESTIMATES, SMALL_ESTIMATES, mergeBothSmall, 1);
-
- costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
- costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
- costEstimator.addLocalMergeCost(BIG_ESTIMATES, BIG_ESTIMATES, mergeBothLarge, 1);
-
-
- assertTrue(mergeBothSmall.compareTo(mergeSmallFirst) < 0);
- assertTrue(mergeBothSmall.compareTo(mergeSmallSecond) < 0);
- assertTrue(mergeSmallFirst.compareTo(mergeSmallSecond) == 0);
- assertTrue(mergeSmallFirst.compareTo(mergeBothLarge) < 0);
- assertTrue(mergeSmallSecond.compareTo(mergeBothLarge) < 0);
-
- // compare merge join and hash join costs
-
- assertTrue(hashBothSmall.compareTo(mergeBothSmall) < 0);
- assertTrue(hashBothLarge.compareTo(mergeBothLarge) < 0);
- assertTrue(hashSmallBuild.compareTo(mergeSmallFirst) < 0);
- assertTrue(hashSmallBuild.compareTo(mergeSmallSecond) < 0);
- assertTrue(hashLargeBuild.compareTo(mergeSmallFirst) < 0);
- assertTrue(hashLargeBuild.compareTo(mergeSmallSecond) < 0);
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testJoinCostFormulasWithWeights() {
- testJoinCostFormulasWithWeights(UNKNOWN_ESTIMATES, SMALL_ESTIMATES);
- testJoinCostFormulasWithWeights(SMALL_ESTIMATES, UNKNOWN_ESTIMATES);
- testJoinCostFormulasWithWeights(UNKNOWN_ESTIMATES, MEDIUM_ESTIMATES);
- testJoinCostFormulasWithWeights(MEDIUM_ESTIMATES, UNKNOWN_ESTIMATES);
- testJoinCostFormulasWithWeights(BIG_ESTIMATES, MEDIUM_ESTIMATES);
- testJoinCostFormulasWithWeights(MEDIUM_ESTIMATES, BIG_ESTIMATES);
- }
-
- private void testJoinCostFormulasWithWeights(EstimateProvider e1, EstimateProvider e2) {
- Costs hf1 = new Costs();
- Costs hf5 = new Costs();
- Costs hs1 = new Costs();
- Costs hs5 = new Costs();
- Costs mm1 = new Costs();
- Costs mm5 = new Costs();
-
- costEstimator.addHybridHashCosts(e1, e2, hf1, 1);
- costEstimator.addHybridHashCosts(e1, e2, hf5, 5);
- costEstimator.addHybridHashCosts(e2, e1, hs1, 1);
- costEstimator.addHybridHashCosts(e2, e1, hs5, 5);
-
- costEstimator.addLocalSortCost(e1, mm1);
- costEstimator.addLocalSortCost(e2, mm1);
- costEstimator.addLocalMergeCost(e1, e2, mm1, 1);
-
- costEstimator.addLocalSortCost(e1, mm5);
- costEstimator.addLocalSortCost(e2, mm5);
- mm5.multiplyWith(5);
- costEstimator.addLocalMergeCost(e1, e2, mm5, 5);
-
- // weight 1 versus weight 5
- assertTrue(hf1.compareTo(hf5) < 0);
- assertTrue(hs1.compareTo(hs5) < 0);
- assertTrue(mm1.compareTo(mm5) < 0);
-
- // hash versus merge
- assertTrue(hf1.compareTo(mm1) < 0);
- assertTrue(hs1.compareTo(mm1) < 0);
- assertTrue(hf5.compareTo(mm5) < 0);
- assertTrue(hs5.compareTo(mm5) < 0);
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Test
- public void testHashJoinCostFormulasWithCaches() {
-
- Costs hashBothUnknown10 = new Costs();
- Costs hashBothUnknownCached10 = new Costs();
-
- Costs hashBothSmall10 = new Costs();
- Costs hashBothSmallCached10 = new Costs();
-
- Costs hashSmallLarge10 = new Costs();
- Costs hashSmallLargeCached10 = new Costs();
-
- Costs hashLargeSmall10 = new Costs();
- Costs hashLargeSmallCached10 = new Costs();
-
- Costs hashLargeSmall1 = new Costs();
- Costs hashLargeSmallCached1 = new Costs();
-
- costEstimator.addHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknown10, 10);
- costEstimator.addCachedHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknownCached10, 10);
-
- costEstimator.addHybridHashCosts(MEDIUM_ESTIMATES, MEDIUM_ESTIMATES, hashBothSmall10, 10);
- costEstimator.addCachedHybridHashCosts(MEDIUM_ESTIMATES, MEDIUM_ESTIMATES, hashBothSmallCached10, 10);
-
- costEstimator.addHybridHashCosts(MEDIUM_ESTIMATES, BIG_ESTIMATES, hashSmallLarge10, 10);
- costEstimator.addCachedHybridHashCosts(MEDIUM_ESTIMATES, BIG_ESTIMATES, hashSmallLargeCached10, 10);
-
- costEstimator.addHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmall10, 10);
- costEstimator.addCachedHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmallCached10, 10);
-
- costEstimator.addHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmall1, 1);
- costEstimator.addCachedHybridHashCosts(BIG_ESTIMATES, MEDIUM_ESTIMATES, hashLargeSmallCached1, 1);
-
- // cached variant is always cheaper
- assertTrue(hashBothUnknown10.compareTo(hashBothUnknownCached10) > 0);
- assertTrue(hashBothSmall10.compareTo(hashBothSmallCached10) > 0);
- assertTrue(hashSmallLarge10.compareTo(hashSmallLargeCached10) > 0);
- assertTrue(hashLargeSmall10.compareTo(hashLargeSmallCached10) > 0);
-
- // caching the large side is better, because then the small one is the one with additional I/O
- assertTrue(hashLargeSmallCached10.compareTo(hashSmallLargeCached10) < 0);
-
- // a weight of one makes the caching the same as the non-cached variant
- assertTrue(hashLargeSmall1.compareTo(hashLargeSmallCached1) == 0);
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Estimate providers
- // --------------------------------------------------------------------------------------------
-
- private static final class UnknownEstimates implements EstimateProvider {
-
- @Override
- public long getEstimatedOutputSize() { return -1; }
-
- @Override
- public long getEstimatedNumRecords() { return -1; }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() { return -1.0f; }
- }
-
- private static final class Estimates implements EstimateProvider {
-
- private final long size;
- private final long records;
- private final float width;
-
- public Estimates(long size, long records) {
- this(size, records, -1.0f);
- }
-
- public Estimates(long size, long records, float width) {
- this.size = size;
- this.records = records;
- this.width = width;
- }
-
- @Override
- public long getEstimatedOutputSize() {
- return this.size;
- }
-
- @Override
- public long getEstimatedNumRecords() {
- return this.records;
- }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() {
- return this.width;
- }
- }
-}
[11/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
new file mode 100644
index 0000000..04bc527
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plantranslate/JobGraphGenerator.java
@@ -0,0 +1,1578 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plantranslate;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.aggregators.AggregatorRegistry;
+import org.apache.flink.api.common.aggregators.AggregatorWithName;
+import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
+import org.apache.flink.api.common.aggregators.LongSumAggregator;
+import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.iterative.convergence.WorksetEmptyConvergenceCriterion;
+import org.apache.flink.runtime.iterative.task.IterationHeadPactTask;
+import org.apache.flink.runtime.iterative.task.IterationIntermediatePactTask;
+import org.apache.flink.runtime.iterative.task.IterationSynchronizationSinkTask;
+import org.apache.flink.runtime.iterative.task.IterationTailPactTask;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.DistributionPattern;
+import org.apache.flink.runtime.jobgraph.InputFormatVertex;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.OutputFormatVertex;
+import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.operators.CoGroupDriver;
+import org.apache.flink.runtime.operators.CoGroupWithSolutionSetFirstDriver;
+import org.apache.flink.runtime.operators.CoGroupWithSolutionSetSecondDriver;
+import org.apache.flink.runtime.operators.DataSinkTask;
+import org.apache.flink.runtime.operators.DataSourceTask;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.JoinWithSolutionSetFirstDriver;
+import org.apache.flink.runtime.operators.JoinWithSolutionSetSecondDriver;
+import org.apache.flink.runtime.operators.MatchDriver;
+import org.apache.flink.runtime.operators.NoOpDriver;
+import org.apache.flink.runtime.operators.RegularPactTask;
+import org.apache.flink.runtime.operators.chaining.ChainedDriver;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Visitor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * This component translates the optimizer's resulting {@link org.apache.flink.optimizer.plan.OptimizedPlan}
+ * to a {@link org.apache.flink.runtime.jobgraph.JobGraph}. The translation is not strictly a one-to-one,
+ * because some nodes from the OptimizedPlan are collapsed into one job vertex.
+ *
+ * This translation does not make any decisions or assumptions. All degrees-of-freedom in the execution
+ * of the job are made by the Optimizer, so that this translation becomes a deterministic mapping.
+ *
+ * The basic method of operation is a top down traversal over the plan graph. On the way down, job vertices
+ * are created for the plan nodes, on the way back up, the nodes connect their predecessors.
+ */
+public class JobGraphGenerator implements Visitor<PlanNode> {
+
+ public static final String MERGE_ITERATION_AUX_TASKS_KEY = "compiler.merge-iteration-aux";
+
+ private static final boolean mergeIterationAuxTasks = GlobalConfiguration.getBoolean(MERGE_ITERATION_AUX_TASKS_KEY, false);
+
+ private static final TaskInChain ALREADY_VISITED_PLACEHOLDER = new TaskInChain(null, null, null);
+
+ // ------------------------------------------------------------------------
+
+ private Map<PlanNode, AbstractJobVertex> vertices; // a map from optimizer nodes to job vertices
+
+ private Map<PlanNode, TaskInChain> chainedTasks; // a map from optimizer nodes to job vertices
+
+ private Map<IterationPlanNode, IterationDescriptor> iterations;
+
+ private List<TaskInChain> chainedTasksInSequence;
+
+ private List<AbstractJobVertex> auxVertices; // auxiliary vertices which are added during job graph generation
+
+ private final int defaultMaxFan;
+
+ private final float defaultSortSpillingThreshold;
+
+ private int iterationIdEnumerator = 1;
+
+ private IterationPlanNode currentIteration; // the current the enclosing iteration
+
+ private List<IterationPlanNode> iterationStack; // stack of enclosing iterations
+
+ private SlotSharingGroup sharingGroup;
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new job graph generator that uses the default values for its resource configuration.
+ */
+ public JobGraphGenerator() {
+ this.defaultMaxFan = ConfigConstants.DEFAULT_SPILLING_MAX_FAN;
+ this.defaultSortSpillingThreshold = ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD;
+ }
+
+ public JobGraphGenerator(Configuration config) {
+ this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY,
+ ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
+ this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
+ ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
+ }
+
+ /**
+ * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a
+ * {@link org.apache.flink.runtime.jobgraph.JobGraph}.
+ *
+ * @param program Optimized plan that is translated into a JobGraph.
+ * @return JobGraph generated frmo the plan.
+ */
+ public JobGraph compileJobGraph(OptimizedPlan program) {
+ this.vertices = new HashMap<PlanNode, AbstractJobVertex>();
+ this.chainedTasks = new HashMap<PlanNode, TaskInChain>();
+ this.chainedTasksInSequence = new ArrayList<TaskInChain>();
+ this.auxVertices = new ArrayList<AbstractJobVertex>();
+ this.iterations = new HashMap<IterationPlanNode, IterationDescriptor>();
+ this.iterationStack = new ArrayList<IterationPlanNode>();
+
+ this.sharingGroup = new SlotSharingGroup();
+
+ // this starts the traversal that generates the job graph
+ program.accept(this);
+
+ // sanity check that we are not somehow in an iteration at the end
+ if (this.currentIteration != null) {
+ throw new CompilerException("The graph translation ended prematurely, leaving an unclosed iteration.");
+ }
+
+ // finalize the iterations
+ for (IterationDescriptor iteration : this.iterations.values()) {
+ if (iteration.getIterationNode() instanceof BulkIterationPlanNode) {
+ finalizeBulkIteration(iteration);
+ } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) {
+ finalizeWorksetIteration(iteration);
+ } else {
+ throw new CompilerException();
+ }
+ }
+
+ // now that the traversal is done, we have the chained tasks write their configs into their
+ // parents' configurations
+ for (TaskInChain tic : this.chainedTasksInSequence) {
+ TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
+ t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
+ }
+
+ // create the job graph object
+ JobGraph graph = new JobGraph(program.getJobName());
+ graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());
+ graph.setAllowQueuedScheduling(false);
+
+ // add vertices to the graph
+ for (AbstractJobVertex vertex : this.vertices.values()) {
+ graph.addVertex(vertex);
+ }
+
+ for (AbstractJobVertex vertex : this.auxVertices) {
+ graph.addVertex(vertex);
+ vertex.setSlotSharingGroup(sharingGroup);
+ }
+
+ // add registered cache file into job configuration
+ for (Entry<String, DistributedCacheEntry> e : program.getOriginalPactPlan().getCachedFiles()) {
+ DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
+ }
+
+ try {
+ InstantiationUtil.writeObjectToConfig(
+ program.getOriginalPactPlan().getExecutionConfig(),
+ graph.getJobConfiguration(),
+ ExecutionConfig.CONFIG_KEY);
+ } catch (IOException e) {
+ throw new RuntimeException("Config object could not be written to Job Configuration: " + e);
+ }
+
+ // release all references again
+ this.vertices = null;
+ this.chainedTasks = null;
+ this.chainedTasksInSequence = null;
+ this.auxVertices = null;
+ this.iterations = null;
+ this.iterationStack = null;
+
+ // return job graph
+ return graph;
+ }
+
+ /**
+ * This methods implements the pre-visiting during a depth-first traversal. It create the job vertex and
+ * sets local strategy.
+ *
+ * @param node
+ * The node that is currently processed.
+ * @return True, if the visitor should descend to the node's children, false if not.
+ * @see org.apache.flink.util.Visitor#preVisit(org.apache.flink.util.Visitable)
+ */
+ @Override
+ public boolean preVisit(PlanNode node) {
+ // check if we have visited this node before. in non-tree graphs, this happens
+ if (this.vertices.containsKey(node) || this.chainedTasks.containsKey(node) || this.iterations.containsKey(node)) {
+ // return false to prevent further descend
+ return false;
+ }
+
+ // the vertex to be created for the current node
+ final AbstractJobVertex vertex;
+ try {
+ if (node instanceof SinkPlanNode) {
+ vertex = createDataSinkVertex((SinkPlanNode) node);
+ }
+ else if (node instanceof SourcePlanNode) {
+ vertex = createDataSourceVertex((SourcePlanNode) node);
+ }
+ else if (node instanceof BulkIterationPlanNode) {
+ BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
+ // for the bulk iteration, we skip creating anything for now. we create the graph
+ // for the step function in the post visit.
+
+ // check that the root of the step function has the same DOP as the iteration.
+ // because the tail must have the same DOP as the head, we can only merge the last
+ // operator with the tail, if they have the same DOP. not merging is currently not
+ // implemented
+ PlanNode root = iterationNode.getRootOfStepFunction();
+ if (root.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("Error: The final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+
+ IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
+ this.iterations.put(iterationNode, descr);
+ vertex = null;
+ }
+ else if (node instanceof WorksetIterationPlanNode) {
+ WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
+
+ // we have the same constraints as for the bulk iteration
+ PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
+ PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
+
+ if (nextWorkSet.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("It is currently not supported that the final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+ if (solutionSetDelta.getParallelism() != node.getParallelism())
+ {
+ throw new CompilerException("It is currently not supported that the final operator of the step " +
+ "function has a different degree of parallelism than the iteration operator itself.");
+ }
+
+ IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
+ this.iterations.put(iterationNode, descr);
+ vertex = null;
+ }
+ else if (node instanceof SingleInputPlanNode) {
+ vertex = createSingleInputVertex((SingleInputPlanNode) node);
+ }
+ else if (node instanceof DualInputPlanNode) {
+ vertex = createDualInputVertex((DualInputPlanNode) node);
+ }
+ else if (node instanceof NAryUnionPlanNode) {
+ // skip the union for now
+ vertex = null;
+ }
+ else if (node instanceof BulkPartialSolutionPlanNode) {
+ // create a head node (or not, if it is merged into its successor)
+ vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
+ }
+ else if (node instanceof SolutionSetPlanNode) {
+ // this represents an access into the solution set index.
+ // we do not create a vertex for the solution set here (we create the head at the workset place holder)
+
+ // we adjust the joins / cogroups that go into the solution set here
+ for (Channel c : node.getOutgoingChannels()) {
+ DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
+ AbstractJobVertex accessingVertex = this.vertices.get(target);
+ TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
+ int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
+
+ // sanity checks
+ if (inputNum == -1) {
+ throw new CompilerException();
+ }
+
+ // adjust the driver
+ if (conf.getDriver().equals(MatchDriver.class)) {
+ conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
+ }
+ else if (conf.getDriver().equals(CoGroupDriver.class)) {
+ conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
+ }
+ else {
+ throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
+ }
+ }
+
+ // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
+ this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
+
+ vertex = null;
+ }
+ else if (node instanceof WorksetPlanNode) {
+ // create the iteration head here
+ vertex = createWorksetIterationHead((WorksetPlanNode) node);
+ }
+ else {
+ throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
+ }
+ }
+ catch (Exception e) {
+ throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
+ }
+
+ // check if a vertex was created, or if it was chained or skipped
+ if (vertex != null) {
+ // set degree of parallelism
+ int pd = node.getParallelism();
+ vertex.setParallelism(pd);
+
+ vertex.setSlotSharingGroup(sharingGroup);
+
+ // check whether this vertex is part of an iteration step function
+ if (this.currentIteration != null) {
+ // check that the task has the same DOP as the iteration as such
+ PlanNode iterationNode = (PlanNode) this.currentIteration;
+ if (iterationNode.getParallelism() < pd) {
+ throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
+ }
+
+ // store the id of the iterations the step functions participate in
+ IterationDescriptor descr = this.iterations.get(this.currentIteration);
+ new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
+ }
+
+ // store in the map
+ this.vertices.put(node, vertex);
+ }
+
+ // returning true causes deeper descend
+ return true;
+ }
+
+ /**
+ * This method implements the post-visit during the depth-first traversal. When the post visit happens,
+ * all of the descendants have been processed, so this method connects all of the current node's
+ * predecessors to the current node.
+ *
+ * @param node
+ * The node currently processed during the post-visit.
+ * @see org.apache.flink.util.Visitor#postVisit(org.apache.flink.util.Visitable) t
+ */
+ @Override
+ public void postVisit(PlanNode node) {
+ try {
+ // --------- check special cases for which we handle post visit differently ----------
+
+ // skip data source node (they have no inputs)
+ // also, do nothing for union nodes, we connect them later when gathering the inputs for a task
+ // solution sets have no input. the initial solution set input is connected when the iteration node is in its postVisit
+ if (node instanceof SourcePlanNode || node instanceof NAryUnionPlanNode || node instanceof SolutionSetPlanNode) {
+ return;
+ }
+
+ // check if we have an iteration. in that case, translate the step function now
+ if (node instanceof IterationPlanNode) {
+ // prevent nested iterations
+ if (node.isOnDynamicPath()) {
+ throw new CompilerException("Nested Iterations are not possible at the moment!");
+ }
+
+ // if we recursively go into an iteration (because the constant path of one iteration contains
+ // another one), we push the current one onto the stack
+ if (this.currentIteration != null) {
+ this.iterationStack.add(this.currentIteration);
+ }
+
+ this.currentIteration = (IterationPlanNode) node;
+ this.currentIteration.acceptForStepFunction(this);
+
+ // pop the current iteration from the stack
+ if (this.iterationStack.isEmpty()) {
+ this.currentIteration = null;
+ } else {
+ this.currentIteration = this.iterationStack.remove(this.iterationStack.size() - 1);
+ }
+
+ // inputs for initial bulk partial solution or initial workset are already connected to the iteration head in the head's post visit.
+ // connect the initial solution set now.
+ if (node instanceof WorksetIterationPlanNode) {
+ // connect the initial solution set
+ WorksetIterationPlanNode wsNode = (WorksetIterationPlanNode) node;
+ AbstractJobVertex headVertex = this.iterations.get(wsNode).getHeadTask();
+ TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ int inputIndex = headConfig.getDriverStrategy().getNumInputs();
+ headConfig.setIterationHeadSolutionSetInputIndex(inputIndex);
+ translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
+ }
+
+ return;
+ }
+
+ final AbstractJobVertex targetVertex = this.vertices.get(node);
+
+
+ // --------- Main Path: Translation of channels ----------
+ //
+ // There are two paths of translation: One for chained tasks (or merged tasks in general),
+ // which do not have their own task vertex. The other for tasks that have their own vertex,
+ // or are the primary task in a vertex (to which the others are chained).
+
+ // check whether this node has its own task, or is merged with another one
+ if (targetVertex == null) {
+ // node's task is merged with another task. it is either chained, of a merged head vertex
+ // from an iteration
+ final TaskInChain chainedTask;
+ if ((chainedTask = this.chainedTasks.get(node)) != null) {
+ // Chained Task. Sanity check first...
+ final Iterator<Channel> inConns = node.getInputs().iterator();
+ if (!inConns.hasNext()) {
+ throw new CompilerException("Bug: Found chained task with no input.");
+ }
+ final Channel inConn = inConns.next();
+
+ if (inConns.hasNext()) {
+ throw new CompilerException("Bug: Found a chained task with more than one input!");
+ }
+ if (inConn.getLocalStrategy() != null && inConn.getLocalStrategy() != LocalStrategy.NONE) {
+ throw new CompilerException("Bug: Found a chained task with an input local strategy.");
+ }
+ if (inConn.getShipStrategy() != null && inConn.getShipStrategy() != ShipStrategyType.FORWARD) {
+ throw new CompilerException("Bug: Found a chained task with an input ship strategy other than FORWARD.");
+ }
+
+ AbstractJobVertex container = chainedTask.getContainingVertex();
+
+ if (container == null) {
+ final PlanNode sourceNode = inConn.getSource();
+ container = this.vertices.get(sourceNode);
+ if (container == null) {
+ // predecessor is itself chained
+ container = this.chainedTasks.get(sourceNode).getContainingVertex();
+ if (container == null) {
+ throw new IllegalStateException("Bug: Chained task predecessor has not been assigned its containing vertex.");
+ }
+ } else {
+ // predecessor is a proper task job vertex and this is the first chained task. add a forward connection entry.
+ new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD);
+ }
+ chainedTask.setContainingVertex(container);
+ }
+
+ // add info about the input serializer type
+ chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0);
+
+ // update name of container task
+ String containerTaskName = container.getName();
+ if(containerTaskName.startsWith("CHAIN ")) {
+ container.setName(containerTaskName+" -> "+chainedTask.getTaskName());
+ } else {
+ container.setName("CHAIN "+containerTaskName+" -> "+chainedTask.getTaskName());
+ }
+
+ this.chainedTasksInSequence.add(chainedTask);
+ return;
+ }
+ else if (node instanceof BulkPartialSolutionPlanNode ||
+ node instanceof WorksetPlanNode)
+ {
+ // merged iteration head task. the task that the head is merged with will take care of it
+ return;
+ } else {
+ throw new CompilerException("Bug: Unrecognized merged task vertex.");
+ }
+ }
+
+ // -------- Here, we translate non-chained tasks -------------
+
+
+ if (this.currentIteration != null) {
+ AbstractJobVertex head = this.iterations.get(this.currentIteration).getHeadTask();
+ // the head may still be null if we descend into the static parts first
+ if (head != null) {
+ targetVertex.setStrictlyCoLocatedWith(head);
+ }
+ }
+
+
+ // create the config that will contain all the description of the inputs
+ final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration());
+
+ // get the inputs. if this node is the head of an iteration, we obtain the inputs from the
+ // enclosing iteration node, because the inputs are the initial inputs to the iteration.
+ final Iterator<Channel> inConns;
+ if (node instanceof BulkPartialSolutionPlanNode) {
+ inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
+ // because the partial solution has its own vertex, is has only one (logical) input.
+ // note this in the task configuration
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
+ } else if (node instanceof WorksetPlanNode) {
+ WorksetPlanNode wspn = (WorksetPlanNode) node;
+ // input that is the initial workset
+ inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
+
+ // because we have a stand-alone (non-merged) workset iteration head, the initial workset will
+ // be input 0 and the solution set will be input 1
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
+ targetVertexConfig.setIterationHeadSolutionSetInputIndex(1);
+ } else {
+ inConns = node.getInputs().iterator();
+ }
+ if (!inConns.hasNext()) {
+ throw new CompilerException("Bug: Found a non-source task with no input.");
+ }
+
+ int inputIndex = 0;
+ while (inConns.hasNext()) {
+ Channel input = inConns.next();
+ inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
+ }
+ // broadcast variables
+ int broadcastInputIndex = 0;
+ for (NamedChannel broadcastInput: node.getBroadcastInputs()) {
+ int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true);
+ targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
+ targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex);
+ broadcastInputIndex += broadcastInputIndexDelta;
+ }
+ } catch (Exception e) {
+ throw new CompilerException(
+ "An error occurred while translating the optimized plan to a nephele JobGraph: " + e.getMessage(), e);
+ }
+ }
+
+ private int translateChannel(Channel input, int inputIndex, AbstractJobVertex targetVertex,
+ TaskConfig targetVertexConfig, boolean isBroadcast) throws Exception
+ {
+ final PlanNode inputPlanNode = input.getSource();
+ final Iterator<Channel> allInChannels;
+
+ if (inputPlanNode instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) inputPlanNode).getListOfInputs().iterator();
+ }
+ else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
+ if (this.vertices.get(inputPlanNode) == null) {
+ // merged iteration head
+ final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
+ final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
+
+ // check if the iteration's input is a union
+ if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs().iterator();
+ } else {
+ allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
+ }
+
+ // also, set the index of the gate with the partial solution
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
+ } else {
+ // standalone iteration head
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+ } else if (inputPlanNode instanceof WorksetPlanNode) {
+ if (this.vertices.get(inputPlanNode) == null) {
+ // merged iteration head
+ final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
+ final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
+
+ // check if the iteration's input is a union
+ if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
+ allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs().iterator();
+ } else {
+ allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
+ }
+
+ // also, set the index of the gate with the partial solution
+ targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
+ } else {
+ // standalone iteration head
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+ } else if (inputPlanNode instanceof SolutionSetPlanNode) {
+ // for now, skip connections with the solution set node, as this is a local index access (later to be parameterized here)
+ // rather than a vertex connection
+ return 0;
+ } else {
+ allInChannels = Collections.singletonList(input).iterator();
+ }
+
+ // check that the type serializer is consistent
+ TypeSerializerFactory<?> typeSerFact = null;
+
+ // accounting for channels on the dynamic path
+ int numChannelsTotal = 0;
+ int numChannelsDynamicPath = 0;
+ int numDynamicSenderTasksTotal = 0;
+
+
+ // expand the channel to all the union channels, in case there is a union operator at its source
+ while (allInChannels.hasNext()) {
+ final Channel inConn = allInChannels.next();
+
+ // sanity check the common serializer
+ if (typeSerFact == null) {
+ typeSerFact = inConn.getSerializer();
+ } else if (!typeSerFact.equals(inConn.getSerializer())) {
+ throw new CompilerException("Conflicting types in union operator.");
+ }
+
+ final PlanNode sourceNode = inConn.getSource();
+ AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
+ TaskConfig sourceVertexConfig;
+
+ if (sourceVertex == null) {
+ // this predecessor is chained to another task or an iteration
+ final TaskInChain chainedTask;
+ final IterationDescriptor iteration;
+ if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
+ // push chained task
+ if (chainedTask.getContainingVertex() == null) {
+ throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
+ }
+ sourceVertex = chainedTask.getContainingVertex();
+ sourceVertexConfig = chainedTask.getTaskConfig();
+ } else if ((iteration = this.iterations.get(sourceNode)) != null) {
+ // predecessor is an iteration
+ sourceVertex = iteration.getHeadTask();
+ sourceVertexConfig = iteration.getHeadFinalResultConfig();
+ } else {
+ throw new CompilerException("Bug: Could not resolve source node for a channel.");
+ }
+ } else {
+ // predecessor is its own vertex
+ sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
+ }
+ DistributionPattern pattern = connectJobVertices(
+ inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
+
+ // accounting on channels and senders
+ numChannelsTotal++;
+ if (inConn.isOnDynamicPath()) {
+ numChannelsDynamicPath++;
+ numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
+ sourceVertex.getParallelism(), targetVertex.getParallelism());
+ }
+ }
+
+ // for the iterations, check that the number of dynamic channels is the same as the number
+ // of channels for this logical input. this condition is violated at the moment, if there
+ // is a union between nodes on the static and nodes on the dynamic path
+ if (numChannelsDynamicPath > 0 && numChannelsTotal != numChannelsDynamicPath) {
+ throw new CompilerException("Error: It is currently not supported to union between dynamic and static path in an iteration.");
+ }
+ if (numDynamicSenderTasksTotal > 0) {
+ if (isBroadcast) {
+ targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
+ } else {
+ targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
+ }
+ }
+
+ // the local strategy is added only once. in non-union case that is the actual edge,
+ // in the union case, it is the edge between union and the target node
+ addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
+ return 1;
+ }
+
+ private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) {
+ if (pattern == DistributionPattern.ALL_TO_ALL) {
+ return numSenders;
+ } else if (pattern == DistributionPattern.POINTWISE) {
+ if (numSenders != numReceivers) {
+ if (numReceivers == 1) {
+ return numSenders;
+ }
+ else if (numSenders == 1) {
+ return 1;
+ }
+ else {
+ throw new CompilerException("Error: A changing degree of parallelism is currently " +
+ "not supported between tasks within an iteration.");
+ }
+ } else {
+ return 1;
+ }
+ } else {
+ throw new CompilerException("Unknown distribution pattern for channels: " + pattern);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Methods for creating individual vertices
+ // ------------------------------------------------------------------------
+
+ private AbstractJobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
+ final String taskName = node.getNodeName();
+ final DriverStrategy ds = node.getDriverStrategy();
+
+ // check, whether chaining is possible
+ boolean chaining = false;
+ {
+ Channel inConn = node.getInput();
+ PlanNode pred = inConn.getSource();
+ chaining = ds.getPushChainDriverClass() != null &&
+ !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged
+ !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways
+ !(pred instanceof WorksetPlanNode) && // workset merges anyways
+ !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
+ inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
+ inConn.getLocalStrategy() == LocalStrategy.NONE &&
+ pred.getOutgoingChannels().size() == 1 &&
+ node.getParallelism() == pred.getParallelism() &&
+ node.getBroadcastInputs().isEmpty();
+
+ // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
+ // in a tail
+ if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
+ node.getOutgoingChannels().size() > 0)
+ {
+ WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
+ if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
+ chaining = false;
+ }
+ }
+ // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
+ if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
+ {
+ BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
+ if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
+ chaining = false;
+ }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
+ wspn.getRootOfTerminationCriterion() == pred)) {
+ chaining = false;
+ }
+ }
+ }
+
+ final AbstractJobVertex vertex;
+ final TaskConfig config;
+
+ if (chaining) {
+ vertex = null;
+ config = new TaskConfig(new Configuration());
+ this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
+ } else {
+ // create task vertex
+ vertex = new AbstractJobVertex(taskName);
+ vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
+
+ config = new TaskConfig(vertex.getConfiguration());
+ config.setDriver(ds.getDriverClass());
+ }
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ // set the driver strategy
+ config.setDriverStrategy(ds);
+ for(int i=0;i<ds.getNumRequiredComparators();i++) {
+ config.setDriverComparator(node.getComparator(i), i);
+ }
+ // assign memory, file-handles, etc.
+ assignDriverResources(node, config);
+ return vertex;
+ }
+
+ private AbstractJobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
+ final String taskName = node.getNodeName();
+ final DriverStrategy ds = node.getDriverStrategy();
+ final AbstractJobVertex vertex = new AbstractJobVertex(taskName);
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+ vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ // set the driver strategy
+ config.setDriver(ds.getDriverClass());
+ config.setDriverStrategy(ds);
+ if (node.getComparator1() != null) {
+ config.setDriverComparator(node.getComparator1(), 0);
+ }
+ if (node.getComparator2() != null) {
+ config.setDriverComparator(node.getComparator2(), 1);
+ }
+ if (node.getPairComparator() != null) {
+ config.setDriverPairComparator(node.getPairComparator());
+ }
+
+ // assign memory, file-handles, etc.
+ assignDriverResources(node, config);
+ return vertex;
+ }
+
+ private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
+ final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName());
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+
+ vertex.setInvokableClass(DataSourceTask.class);
+ vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ config.setOutputSerializer(node.getSerializer());
+ return vertex;
+ }
+
+ private AbstractJobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
+ final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName());
+ final TaskConfig config = new TaskConfig(vertex.getConfiguration());
+
+ vertex.setInvokableClass(DataSinkTask.class);
+ vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));
+
+ // set user code
+ config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
+ config.setStubParameters(node.getProgramOperator().getParameters());
+
+ return vertex;
+ }
+
+ private AbstractJobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
+ // get the bulk iteration that corresponds to this partial solution node
+ final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
+
+ // check whether we need an individual vertex for the partial solution, or whether we
+ // attach ourselves to the vertex of the parent node. We can combine the head with a node of
+ // the step function, if
+ // 1) There is one parent that the partial solution connects to via a forward pattern and no
+ // local strategy
+ // 2) DOP and the number of subtasks per instance does not change
+ // 3) That successor is not a union
+ // 4) That successor is not itself the last node of the step function
+ // 5) There is no local strategy on the edge for the initial partial solution, as
+ // this translates to a local strategy that would only be executed in the first iteration
+
+ final boolean merge;
+ if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
+ final Channel c = pspn.getOutgoingChannels().get(0);
+ final PlanNode successor = c.getTarget();
+ merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
+ c.getLocalStrategy() == LocalStrategy.NONE &&
+ c.getTempMode() == TempMode.NONE &&
+ successor.getParallelism() == pspn.getParallelism() &&
+ !(successor instanceof NAryUnionPlanNode) &&
+ successor != iteration.getRootOfStepFunction() &&
+ iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
+ } else {
+ merge = false;
+ }
+
+ // create or adopt the head vertex
+ final AbstractJobVertex toReturn;
+ final AbstractJobVertex headVertex;
+ final TaskConfig headConfig;
+ if (merge) {
+ final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
+ headVertex = (AbstractJobVertex) this.vertices.get(successor);
+
+ if (headVertex == null) {
+ throw new CompilerException(
+ "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
+ }
+
+ // reset the vertex type to iteration head
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ toReturn = null;
+ } else {
+ // instantiate the head vertex and give it a no-op driver as the driver strategy.
+ // everything else happens in the post visit, after the input (the initial partial solution)
+ // is connected.
+ headVertex = new AbstractJobVertex("PartialSolution ("+iteration.getNodeName()+")");
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ headConfig.setDriver(NoOpDriver.class);
+ toReturn = headVertex;
+ }
+
+ // create the iteration descriptor and the iteration to it
+ IterationDescriptor descr = this.iterations.get(iteration);
+ if (descr == null) {
+ throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
+ }
+ descr.setHeadTask(headVertex, headConfig);
+
+ return toReturn;
+ }
+
+ private AbstractJobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
+ // get the bulk iteration that corresponds to this partial solution node
+ final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
+
+ // check whether we need an individual vertex for the partial solution, or whether we
+ // attach ourselves to the vertex of the parent node. We can combine the head with a node of
+ // the step function, if
+ // 1) There is one parent that the partial solution connects to via a forward pattern and no
+ // local strategy
+ // 2) DOP and the number of subtasks per instance does not change
+ // 3) That successor is not a union
+ // 4) That successor is not itself the last node of the step function
+ // 5) There is no local strategy on the edge for the initial workset, as
+ // this translates to a local strategy that would only be executed in the first superstep
+
+ final boolean merge;
+ if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
+ final Channel c = wspn.getOutgoingChannels().get(0);
+ final PlanNode successor = c.getTarget();
+ merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
+ c.getLocalStrategy() == LocalStrategy.NONE &&
+ c.getTempMode() == TempMode.NONE &&
+ successor.getParallelism() == wspn.getParallelism() &&
+ !(successor instanceof NAryUnionPlanNode) &&
+ successor != iteration.getNextWorkSetPlanNode() &&
+ iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
+ } else {
+ merge = false;
+ }
+
+ // create or adopt the head vertex
+ final AbstractJobVertex toReturn;
+ final AbstractJobVertex headVertex;
+ final TaskConfig headConfig;
+ if (merge) {
+ final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
+ headVertex = (AbstractJobVertex) this.vertices.get(successor);
+
+ if (headVertex == null) {
+ throw new CompilerException(
+ "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
+ }
+
+ // reset the vertex type to iteration head
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ toReturn = null;
+ } else {
+ // instantiate the head vertex and give it a no-op driver as the driver strategy.
+ // everything else happens in the post visit, after the input (the initial partial solution)
+ // is connected.
+ headVertex = new AbstractJobVertex("IterationHead("+iteration.getNodeName()+")");
+ headVertex.setInvokableClass(IterationHeadPactTask.class);
+ headConfig = new TaskConfig(headVertex.getConfiguration());
+ headConfig.setDriver(NoOpDriver.class);
+ toReturn = headVertex;
+ }
+
+ headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
+
+ // create the iteration descriptor and the iteration to it
+ IterationDescriptor descr = this.iterations.get(iteration);
+ if (descr == null) {
+ throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
+ }
+ descr.setHeadTask(headVertex, headConfig);
+
+ return toReturn;
+ }
+
+ private void assignDriverResources(PlanNode node, TaskConfig config) {
+ final double relativeMem = node.getRelativeMemoryPerSubTask();
+ if (relativeMem > 0) {
+ config.setRelativeMemoryDriver(relativeMem);
+ config.setFilehandlesDriver(this.defaultMaxFan);
+ config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
+ }
+ }
+
+ private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) {
+ if (c.getRelativeMemoryLocalStrategy() > 0) {
+ config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy());
+ config.setFilehandlesInput(inputNum, this.defaultMaxFan);
+ config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Connecting Vertices
+ // ------------------------------------------------------------------------
+
+ /**
+ * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
+ * channel is then the channel into the union node, the local strategy channel the one from the union to the
+ * actual target operator.
+ *
+ * @param channel
+ * @param inputNumber
+ * @param sourceVertex
+ * @param sourceConfig
+ * @param targetVertex
+ * @param targetConfig
+ * @param isBroadcast
+ * @throws CompilerException
+ */
+ private DistributionPattern connectJobVertices(Channel channel, int inputNumber,
+ final AbstractJobVertex sourceVertex, final TaskConfig sourceConfig,
+ final AbstractJobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast)
+ throws CompilerException
+ {
+ // ------------ connect the vertices to the job graph --------------
+ final DistributionPattern distributionPattern;
+
+ switch (channel.getShipStrategy()) {
+ case FORWARD:
+ distributionPattern = DistributionPattern.POINTWISE;
+ break;
+ case PARTITION_RANDOM:
+ case BROADCAST:
+ case PARTITION_HASH:
+ case PARTITION_CUSTOM:
+ case PARTITION_RANGE:
+ case PARTITION_FORCED_REBALANCE:
+ distributionPattern = DistributionPattern.ALL_TO_ALL;
+ break;
+ default:
+ throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
+ }
+
+ final ResultPartitionType resultType;
+
+ switch (channel.getDataExchangeMode()) {
+
+ case PIPELINED:
+ resultType = ResultPartitionType.PIPELINED;
+ break;
+
+ case BATCH:
+ // BLOCKING results are currently not supported in closed loop iterations
+ //
+ // See https://issues.apache.org/jira/browse/FLINK-1713 for details
+ resultType = channel.getSource().isOnDynamicPath()
+ ? ResultPartitionType.PIPELINED
+ : ResultPartitionType.BLOCKING;
+ break;
+
+ case PIPELINE_WITH_BATCH_FALLBACK:
+ throw new UnsupportedOperationException("Data exchange mode " +
+ channel.getDataExchangeMode() + " currently not supported.");
+
+ default:
+ throw new UnsupportedOperationException("Unknown data exchange mode.");
+
+ }
+
+ targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
+
+ // -------------- configure the source task's ship strategy strategies in task config --------------
+ final int outputIndex = sourceConfig.getNumOutputs();
+ sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
+ if (outputIndex == 0) {
+ sourceConfig.setOutputSerializer(channel.getSerializer());
+ }
+ if (channel.getShipStrategyComparator() != null) {
+ sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
+ }
+
+ if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
+
+ final DataDistribution dataDistribution = channel.getDataDistribution();
+ if (dataDistribution != null) {
+ sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
+ } else {
+ throw new RuntimeException("Range partitioning requires data distribution");
+ // TODO: inject code and configuration for automatic histogram generation
+ }
+ }
+
+ if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
+ if (channel.getPartitioner() != null) {
+ sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
+ } else {
+ throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
+ }
+ }
+
+ // ---------------- configure the receiver -------------------
+ if (isBroadcast) {
+ targetConfig.addBroadcastInputToGroup(inputNumber);
+ } else {
+ targetConfig.addInputToGroup(inputNumber);
+ }
+ return distributionPattern;
+ }
+
+ private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
+ // serializer
+ if (isBroadcastChannel) {
+ config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
+
+ if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
+ throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
+ } else {
+ return;
+ }
+ } else {
+ config.setInputSerializer(channel.getSerializer(), inputNum);
+ }
+
+ // local strategy
+ if (channel.getLocalStrategy() != LocalStrategy.NONE) {
+ config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
+ if (channel.getLocalStrategyComparator() != null) {
+ config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
+ }
+ }
+
+ assignLocalStrategyResources(channel, config, inputNum);
+
+ // materialization / caching
+ if (channel.getTempMode() != null) {
+ final TempMode tm = channel.getTempMode();
+
+ boolean needsMemory = false;
+ // Don't add a pipeline breaker if the data exchange is already blocking.
+ if (tm.breaksPipeline() && channel.getDataExchangeMode() != DataExchangeMode.BATCH) {
+ config.setInputAsynchronouslyMaterialized(inputNum, true);
+ needsMemory = true;
+ }
+ if (tm.isCached()) {
+ config.setInputCached(inputNum, true);
+ needsMemory = true;
+ }
+
+ if (needsMemory) {
+ // sanity check
+ if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
+ throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
+ }
+ config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
+ }
+ }
+ }
+
+ private void finalizeBulkIteration(IterationDescriptor descr) {
+
+ final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
+ final AbstractJobVertex headVertex = descr.getHeadTask();
+ final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
+
+ // ------------ finalize the head config with the final outputs and the sync gate ------------
+ final int numStepFunctionOuts = headConfig.getNumOutputs();
+ final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
+
+ if (numStepFunctionOuts == 0) {
+ throw new CompilerException("The iteration has no operation inside the step function.");
+ }
+
+ headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
+ headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
+ final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
+ if (relativeMemForBackChannel <= 0) {
+ throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
+ }
+ headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
+
+ // --------------------------- create the sync task ---------------------------
+ final AbstractJobVertex sync = new AbstractJobVertex("Sync(" + bulkNode.getNodeName() + ")");
+ sync.setInvokableClass(IterationSynchronizationSinkTask.class);
+ sync.setParallelism(1);
+ this.auxVertices.add(sync);
+
+ final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
+ syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
+
+ // set the number of iteration / convergence criterion for the sync
+ final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
+ if (maxNumIterations < 1) {
+ throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
+ }
+ syncConfig.setNumberOfIterations(maxNumIterations);
+
+ // connect the sync task
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+
+ // ----------------------------- create the iteration tail ------------------------------
+
+ final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
+ final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
+ final TaskConfig tailConfig;
+
+ AbstractJobVertex rootOfStepFunctionVertex = (AbstractJobVertex) this.vertices.get(rootOfStepFunction);
+ if (rootOfStepFunctionVertex == null) {
+ // last op is chained
+ final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
+ }
+ rootOfStepFunctionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+
+ // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
+ tailConfig = taskInChain.getTaskConfig();
+ } else {
+ tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
+ }
+
+ tailConfig.setIsWorksetUpdate();
+
+ // No following termination criterion
+ if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
+
+ rootOfStepFunctionVertex.setInvokableClass(IterationTailPactTask.class);
+
+ tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
+ }
+
+
+ // create the fake output task for termination criterion, if needed
+ final TaskConfig tailConfigOfTerminationCriterion;
+ // If we have a termination criterion and it is not an intermediate node
+ if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
+ AbstractJobVertex rootOfTerminationCriterionVertex = (AbstractJobVertex) this.vertices.get(rootOfTerminationCriterion);
+
+
+ if (rootOfTerminationCriterionVertex == null) {
+ // last op is chained
+ final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
+ }
+ rootOfTerminationCriterionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+
+ // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
+ tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
+ } else {
+ tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
+ }
+
+ rootOfTerminationCriterionVertex.setInvokableClass(IterationTailPactTask.class);
+ // Hack
+ tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
+ tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
+
+ // tell the head that it needs to wait for the solution set updates
+ headConfig.setWaitForSolutionSetUpdate();
+ }
+
+ // ------------------- register the aggregators -------------------
+ AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
+ Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
+
+ headConfig.addIterationAggregators(allAggregators);
+ syncConfig.addIterationAggregators(allAggregators);
+
+ String convAggName = aggs.getConvergenceCriterionAggregatorName();
+ ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
+
+ if (convCriterion != null || convAggName != null) {
+ if (convCriterion == null) {
+ throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
+ }
+ if (convAggName == null) {
+ throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
+ }
+
+ syncConfig.setConvergenceCriterion(convAggName, convCriterion);
+ }
+ }
+
+ private void finalizeWorksetIteration(IterationDescriptor descr) {
+ final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
+ final AbstractJobVertex headVertex = descr.getHeadTask();
+ final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
+ final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
+
+ // ------------ finalize the head config with the final outputs and the sync gate ------------
+ {
+ final int numStepFunctionOuts = headConfig.getNumOutputs();
+ final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
+
+ if (numStepFunctionOuts == 0) {
+ throw new CompilerException("The workset iteration has no operation on the workset inside the step function.");
+ }
+
+ headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
+ headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
+ final double relativeMemory = iterNode.getRelativeMemoryPerSubTask();
+ if (relativeMemory <= 0) {
+ throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
+ }
+
+ headConfig.setIsWorksetIteration();
+ headConfig.setRelativeBackChannelMemory(relativeMemory / 2);
+ headConfig.setRelativeSolutionSetMemory(relativeMemory / 2);
+
+ // set the solution set serializer and comparator
+ headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
+ headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
+ }
+
+ // --------------------------- create the sync task ---------------------------
+ final TaskConfig syncConfig;
+ {
+ final AbstractJobVertex sync = new AbstractJobVertex("Sync (" + iterNode.getNodeName() + ")");
+ sync.setInvokableClass(IterationSynchronizationSinkTask.class);
+ sync.setParallelism(1);
+ this.auxVertices.add(sync);
+
+ syncConfig = new TaskConfig(sync.getConfiguration());
+ syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
+
+ // set the number of iteration / convergence criterion for the sync
+ final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
+ if (maxNumIterations < 1) {
+ throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
+ }
+ syncConfig.setNumberOfIterations(maxNumIterations);
+
+ // connect the sync task
+ sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
+ }
+
+ // ----------------------------- create the iteration tails -----------------------------
+ // ----------------------- for next workset and solution set delta-----------------------
+
+ {
+ // we have three possible cases:
+ // 1) Two tails, one for workset update, one for solution set update
+ // 2) One tail for workset update, solution set update happens in an intermediate task
+ // 3) One tail for solution set update, workset update happens in an intermediate task
+
+ final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
+ final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
+
+ final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
+ final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
+
+ {
+ // get the vertex for the workset update
+ final TaskConfig worksetTailConfig;
+ AbstractJobVertex nextWorksetVertex = (AbstractJobVertex) this.vertices.get(nextWorksetNode);
+ if (nextWorksetVertex == null) {
+ // nextWorksetVertex is chained
+ TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
+ }
+ nextWorksetVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+ worksetTailConfig = taskInChain.getTaskConfig();
+ } else {
+ worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
+ }
+
+ // mark the node to perform workset updates
+ worksetTailConfig.setIsWorksetIteration();
+ worksetTailConfig.setIsWorksetUpdate();
+
+ if (hasWorksetTail) {
+ nextWorksetVertex.setInvokableClass(IterationTailPactTask.class);
+
+ worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
+ }
+ }
+ {
+ final TaskConfig solutionDeltaConfig;
+ AbstractJobVertex solutionDeltaVertex = (AbstractJobVertex) this.vertices.get(solutionDeltaNode);
+ if (solutionDeltaVertex == null) {
+ // last op is chained
+ TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
+ if (taskInChain == null) {
+ throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
+ }
+ solutionDeltaVertex = (AbstractJobVertex) taskInChain.getContainingVertex();
+ solutionDeltaConfig = taskInChain.getTaskConfig();
+ } else {
+ solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
+ }
+
+ solutionDeltaConfig.setIsWorksetIteration();
+ solutionDeltaConfig.setIsSolutionSetUpdate();
+
+ if (hasSolutionSetTail) {
+ solutionDeltaVertex.setInvokableClass(IterationTailPactTask.class);
+
+ solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
+
+ // tell the head that it needs to wait for the solution set updates
+ headConfig.setWaitForSolutionSetUpdate();
+ }
+ else {
+ // no tail, intermediate update. must be immediate update
+ if (!iterNode.isImmediateSolutionSetUpdate()) {
+ throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates.");
+ }
+ solutionDeltaConfig.setIsSolutionSetUpdateWithoutReprobe();
+ }
+ }
+ }
+
+ // ------------------- register the aggregators -------------------
+ AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
+ Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
+
+ for (AggregatorWithName<?> agg : allAggregators) {
+ if (agg.getName().equals(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME)) {
+ throw new CompilerException("User defined aggregator used the same name as built-in workset " +
+ "termination check aggregator: " + WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME);
+ }
+ }
+
+ headConfig.addIterationAggregators(allAggregators);
+ syncConfig.addIterationAggregators(allAggregators);
+
+ String convAggName = aggs.getConvergenceCriterionAggregatorName();
+ ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
+
+ if (convCriterion != null || convAggName != null) {
+ throw new CompilerException("Error: Cannot use custom convergence criterion with workset iteration. Workset iterations have implicit convergence criterion where workset is empty.");
+ }
+
+ headConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
+ syncConfig.addIterationAggregator(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new LongSumAggregator());
+ syncConfig.setConvergenceCriterion(WorksetEmptyConvergenceCriterion.AGGREGATOR_NAME, new WorksetEmptyConvergenceCriterion());
+ }
+
+ private static String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
+ try {
+ if (wrapper.hasObject()) {
+ try {
+ return wrapper.getUserCodeObject().toString();
+ }
+ catch (Throwable t) {
+ return wrapper.getUserCodeClass().getName();
+ }
+ }
+ else {
+ return wrapper.getUserCodeClass().getName();
+ }
+ }
+ catch (Throwable t) {
+ return null;
+ }
+ }
+
+ // -------------------------------------------------------------------------------------
+ // Descriptors for tasks / configurations that are chained or merged with other tasks
+ // -------------------------------------------------------------------------------------
+
+ /**
+ * Utility class that describes a task in a sequence of chained tasks. Chained tasks are tasks that run
+ * together in one thread.
+ */
+ private static final class TaskInChain {
+
+ private final Class<? extends ChainedDriver<?, ?>> chainedTask;
+
+ private final TaskConfig taskConfig;
+
+ private final String taskName;
+
+ private AbstractJobVertex containingVertex;
+
+ TaskInChain(Class<? extends ChainedDriver<?, ?>> chainedTask, TaskConfig taskConfig,
+ String taskName) {
+ this.chainedTask = chainedTask;
+ this.taskConfig = taskConfig;
+ this.taskName = taskName;
+ }
+
+ public Class<? extends ChainedDriver<?, ?>> getChainedTask() {
+ return this.chainedTask;
+ }
+
+ public TaskConfig getTaskConfig() {
+ return this.taskConfig;
+ }
+
+ public String getTaskName() {
+ return this.taskName;
+ }
+
+ public AbstractJobVertex getContainingVertex() {
+ return this.containingVertex;
+ }
+
+ public void setContainingVertex(AbstractJobVertex containingVertex) {
+ this.containingVertex = containingVertex;
+ }
+ }
+
+ private static final class IterationDescriptor {
+
+ private final IterationPlanNode iterationNode;
+
+ private AbstractJobVertex headTask;
+
+ private TaskConfig headConfig;
+
+ private TaskConfig headFinalResultConfig;
+
+ private final int id;
+
+ public IterationDescriptor(IterationPlanNode iterationNode, int id) {
+ this.iterationNode = iterationNode;
+ this.id = id;
+ }
+
+ public IterationPlanNode getIterationNode() {
+ return iterationNode;
+ }
+
+ public void setHeadTask(AbstractJobVertex headTask, TaskConfig headConfig) {
+ this.headTask = headTask;
+ this.headFinalResultConfig = new TaskConfig(new Configuration());
+
+ // check if we already had a configuration, for example if the solution set was
+ if (this.headConfig != null) {
+ headConfig.getConfiguration().addAll(this.headConfig.getConfiguration());
+ }
+
+ this.headConfig = headConfig;
+ }
+
+ public AbstractJobVertex getHeadTask() {
+ return headTask;
+ }
+
+ public TaskConfig getHeadFinalResultConfig() {
+ return headFinalResultConfig;
+ }
+
+ public int getId() {
+ return this.id;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
new file mode 100644
index 0000000..f2b736c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/AbstractSchema.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.postpass;
+
+import java.util.Map;
+
+public abstract class AbstractSchema<X> implements Iterable<Map.Entry<Integer, X>> {
+
+ private int numConnectionsThatContributed;
+
+
+ public int getNumConnectionsThatContributed() {
+ return this.numConnectionsThatContributed;
+ }
+
+ public void increaseNumConnectionsThatContributed() {
+ this.numConnectionsThatContributed++;
+ }
+
+ public abstract void addType(int pos, X type) throws ConflictingFieldTypeInfoException;
+
+ public abstract X getType(int field);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
new file mode 100644
index 0000000..56d914c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/postpass/ConflictingFieldTypeInfoException.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.postpass;
+
+public final class ConflictingFieldTypeInfoException extends Exception {
+
+ private static final long serialVersionUID = 3991352502693288321L;
+
+ private final int fieldNumber;
+
+ private final Object previousType, newType;
+
+
+ public ConflictingFieldTypeInfoException(int fieldNumber, Object previousType, Object newType) {
+ super("Conflicting type info for field " + fieldNumber + ": Old='" + previousType + "', new='" + newType + "'.");
+ this.fieldNumber = fieldNumber;
+ this.previousType = previousType;
+ this.newType = newType;
+ }
+
+
+ public int getFieldNumber() {
+ return fieldNumber;
+ }
+
+ public Object getPreviousType() {
+ return this.previousType;
+ }
+
+ public Object getNewType() {
+ return this.newType;
+ }
+}
[18/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
new file mode 100644
index 0000000..cbd58ca
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/JoinNode.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.operators.AbstractJoinDescriptor;
+import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
+import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.operators.SortMergeJoinDescriptor;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * The Optimizer representation of a join operator.
+ */
+public class JoinNode extends TwoInputNode {
+
+ private List<OperatorDescriptorDual> dataProperties;
+
+ /**
+ * Creates a new JoinNode for the given join operator.
+ *
+ * @param joinOperatorBase The join operator object.
+ */
+ public JoinNode(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase) {
+ super(joinOperatorBase);
+
+ this.dataProperties = getDataProperties(joinOperatorBase,
+ joinOperatorBase.getJoinHint(), joinOperatorBase.getCustomPartitioner());
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the contract object for this match node.
+ *
+ * @return The contract.
+ */
+ @Override
+ public JoinOperatorBase<?, ?, ?, ?> getOperator() {
+ return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Join";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return this.dataProperties;
+ }
+
+ public void makeJoinWithSolutionSet(int solutionsetInputIndex) {
+ OperatorDescriptorDual op;
+ if (solutionsetInputIndex == 0) {
+ op = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
+ } else if (solutionsetInputIndex == 1) {
+ op = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ this.dataProperties = Collections.singletonList(op);
+ }
+
+ /**
+ * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
+ * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
+ * The result cardinality is hence the larger one.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
+ long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
+ this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
+
+ if (this.estimatedNumRecords >= 0) {
+ float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
+
+ if (width > 0) {
+ this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
+ }
+ }
+ }
+
+ private List<OperatorDescriptorDual> getDataProperties(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint,
+ Partitioner<?> customPartitioner)
+ {
+ // see if an internal hint dictates the strategy to use
+ Configuration conf = joinOperatorBase.getParameters();
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
+
+ if (localStrategy != null) {
+ final AbstractJoinDescriptor fixedDriverStrat;
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
+ {
+ fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
+ }
+ else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
+ fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
+ }
+ else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
+ fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
+ }
+ else {
+ throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
+ }
+
+ if (customPartitioner != null) {
+ fixedDriverStrat.setCustomPartitioner(customPartitioner);
+ }
+
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+ list.add(fixedDriverStrat);
+ return list;
+ }
+ else {
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+
+ joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint;
+
+ switch (joinHint) {
+ case BROADCAST_HASH_FIRST:
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, true, false, false));
+ break;
+ case BROADCAST_HASH_SECOND:
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, true, false));
+ break;
+ case REPARTITION_HASH_FIRST:
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, false, false, true));
+ break;
+ case REPARTITION_HASH_SECOND:
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, false, true));
+ break;
+ case REPARTITION_SORT_MERGE:
+ list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2, false, false, true));
+ break;
+ case OPTIMIZER_CHOOSES:
+ list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2));
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2));
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2));
+ break;
+ default:
+ throw new CompilerException("Unrecognized join hint: " + joinHint);
+ }
+
+ if (customPartitioner != null) {
+ for (OperatorDescriptorDual descr : list) {
+ ((AbstractJoinDescriptor) descr).setCustomPartitioner(customPartitioner);
+ }
+ }
+
+ return list;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
new file mode 100644
index 0000000..35def59
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapNode.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.MapDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+/**
+ * The optimizer's internal representation of a <i>Map</i> operator node.
+ */
+public class MapNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ /**
+ * Creates a new MapNode for the given operator.
+ *
+ * @param operator The map operator.
+ */
+ public MapNode(SingleInputOperator<?, ?, ?> operator) {
+ super(operator);
+
+ this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new MapDescriptor());
+ }
+
+ @Override
+ public String getName() {
+ return "Map";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ /**
+ * Computes the estimates for the Map operator.
+ * We assume that by default, Map takes one value and transforms it into another value.
+ * The cardinality consequently stays the same.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
new file mode 100644
index 0000000..b287c33
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MapPartitionNode.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.MapPartitionDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+/**
+ * The optimizer's internal representation of a <i>MapPartition</i> operator node.
+ */
+public class MapPartitionNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ /**
+ * Creates a new MapNode for the given contract.
+ *
+ * @param operator The map partition contract object.
+ */
+ public MapPartitionNode(SingleInputOperator<?, ?, ?> operator) {
+ super(operator);
+
+ this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new MapPartitionDescriptor());
+ }
+
+ @Override
+ public String getName() {
+ return "MapPartition";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ /**
+ * Computes the estimates for the MapPartition operator.
+ * We assume that by default, Map takes one value and transforms it into another value.
+ * The cardinality consequently stays the same.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // we really cannot make any estimates here
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
new file mode 100644
index 0000000..de3cd22
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/MatchNode.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.operators.HashJoinBuildFirstProperties;
+import org.apache.flink.optimizer.operators.HashJoinBuildSecondProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.operators.SortMergeJoinDescriptor;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * The Optimizer representation of a join operator.
+ */
+public class MatchNode extends TwoInputNode {
+
+ private List<OperatorDescriptorDual> dataProperties;
+
+ /**
+ * Creates a new MatchNode for the given join operator.
+ *
+ * @param joinOperatorBase The join operator object.
+ */
+ public MatchNode(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase) {
+ super(joinOperatorBase);
+ this.dataProperties = getDataProperties(joinOperatorBase, joinOperatorBase.getJoinHint());
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the contract object for this match node.
+ *
+ * @return The contract.
+ */
+ @Override
+ public JoinOperatorBase<?, ?, ?, ?> getOperator() {
+ return (JoinOperatorBase<?, ?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Join";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return this.dataProperties;
+ }
+
+ public void makeJoinWithSolutionSet(int solutionsetInputIndex) {
+ OperatorDescriptorDual op;
+ if (solutionsetInputIndex == 0) {
+ op = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
+ } else if (solutionsetInputIndex == 1) {
+ op = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ this.dataProperties = Collections.singletonList(op);
+ }
+
+ /**
+ * The default estimates build on the principle of inclusion: The smaller input key domain is included in the larger
+ * input key domain. We also assume that every key from the larger input has one join partner in the smaller input.
+ * The result cardinality is hence the larger one.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
+ long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
+ this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : Math.max(card1, card2);
+
+ if (this.estimatedNumRecords >= 0) {
+ float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
+
+ if (width > 0) {
+ this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
+ }
+ }
+ }
+
+ private List<OperatorDescriptorDual> getDataProperties(JoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint) {
+ // see if an internal hint dictates the strategy to use
+ Configuration conf = joinOperatorBase.getParameters();
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
+
+ if (localStrategy != null) {
+ final OperatorDescriptorDual fixedDriverStrat;
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) ||
+ Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy) )
+ {
+ fixedDriverStrat = new SortMergeJoinDescriptor(this.keys1, this.keys2);
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
+ fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
+ fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
+ } else {
+ throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
+ }
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+ list.add(fixedDriverStrat);
+ return list;
+ }
+ else {
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+
+ joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint;
+
+ switch (joinHint) {
+ case BROADCAST_HASH_FIRST:
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, true, false, false));
+ break;
+ case BROADCAST_HASH_SECOND:
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, true, false));
+ break;
+ case REPARTITION_HASH_FIRST:
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, false, false, true));
+ break;
+ case REPARTITION_HASH_SECOND:
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, false, true));
+ break;
+ case REPARTITION_SORT_MERGE:
+ list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2, false, false, true));
+ break;
+ case OPTIMIZER_CHOOSES:
+ list.add(new SortMergeJoinDescriptor(this.keys1, this.keys2));
+ list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2));
+ list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2));
+ break;
+ default:
+ throw new CompilerException("Unrecognized join hint: " + joinHint);
+ }
+
+ return list;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
new file mode 100644
index 0000000..76467cf
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/NoOpNode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.NoOpDescriptor;
+
+/**
+ * The optimizer's internal representation of a <i>No Operation</i> node.
+ */
+public class NoOpNode extends UnaryOperatorNode {
+
+ public NoOpNode() {
+ super("No Op", new FieldSet(), new NoOpDescriptor());
+ }
+
+ public NoOpNode(String name) {
+ super(name, new FieldSet(), new NoOpDescriptor());
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
new file mode 100644
index 0000000..0cad34e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
@@ -0,0 +1,1172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.AbstractUdfOperator;
+import org.apache.flink.api.common.operators.CompilerHints;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plandump.DumpableConnection;
+import org.apache.flink.optimizer.plandump.DumpableNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitable;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The OptimizerNode is the base class of all nodes in the optimizer DAG. The optimizer DAG is the
+ * optimizer's representation of a program, created before the actual optimization (which creates different
+ * candidate plans and computes their cost).
+ * <p>>
+ * Nodes in the DAG correspond (almost) one-to-one to the operators in a program. The optimizer DAG is constructed
+ * to hold the additional information that the optimizer needs:
+ * <ul>
+ * <li>Estimates of the data size processed by each operator</li>
+ * <li>Helper structures to track where the data flow "splits" and "joins", to support flows that are
+ * DAGs but not trees.</li>
+ * <li>Tags and weights to differentiate between loop-variant and -invariant parts of an iteration</li>
+ * <li>Interesting properties to be used during the enumeration of candidate plans</li>
+ * </ul>
+ */
+public abstract class OptimizerNode implements Visitable<OptimizerNode>, EstimateProvider, DumpableNode<OptimizerNode> {
+
+ public static final int MAX_DYNAMIC_PATH_COST_WEIGHT = 100;
+
+ // --------------------------------------------------------------------------------------------
+ // Members
+ // --------------------------------------------------------------------------------------------
+
+ private final Operator<?> operator; // The operator (Reduce / Join / DataSource / ...)
+
+ private List<String> broadcastConnectionNames = new ArrayList<String>(); // the broadcast inputs names of this node
+
+ private List<DagConnection> broadcastConnections = new ArrayList<DagConnection>(); // the broadcast inputs of this node
+
+ private List<DagConnection> outgoingConnections; // The links to succeeding nodes
+
+ private InterestingProperties intProps; // the interesting properties of this node
+
+ // --------------------------------- Branch Handling ------------------------------------------
+
+ protected List<UnclosedBranchDescriptor> openBranches; // stack of branches in the sub-graph that are not joined
+
+ protected Set<OptimizerNode> closedBranchingNodes; // stack of branching nodes which have already been closed
+
+ protected List<OptimizerNode> hereJoinedBranches; // the branching nodes (node with multiple outputs)
+ // that are partially joined (through multiple inputs or broadcast vars)
+
+ // ---------------------------- Estimates and Annotations -------------------------------------
+
+ protected long estimatedOutputSize = -1; // the estimated size of the output (bytes)
+
+ protected long estimatedNumRecords = -1; // the estimated number of key/value pairs in the output
+
+ protected Set<FieldSet> uniqueFields; // set of attributes that will always be unique after this node
+
+ // --------------------------------- General Parameters ---------------------------------------
+
+ private int parallelism = -1; // the number of parallel instances of this node
+
+ private long minimalMemoryPerSubTask = -1;
+
+ protected int id = -1; // the id for this node.
+
+ protected int costWeight = 1; // factor to weight the costs for dynamic paths
+
+ protected boolean onDynamicPath;
+
+ protected List<PlanNode> cachedPlans; // cache candidates, because the may be accessed repeatedly
+
+ // ------------------------------------------------------------------------
+ // Constructor / Setup
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new optimizer node that represents the given program operator.
+ *
+ * @param op The operator that the node represents.
+ */
+ public OptimizerNode(Operator<?> op) {
+ this.operator = op;
+ readStubAnnotations();
+ }
+
+ protected OptimizerNode(OptimizerNode toCopy) {
+ this.operator = toCopy.operator;
+ this.intProps = toCopy.intProps;
+
+ this.openBranches = toCopy.openBranches;
+ this.closedBranchingNodes = toCopy.closedBranchingNodes;
+
+ this.estimatedOutputSize = toCopy.estimatedOutputSize;
+ this.estimatedNumRecords = toCopy.estimatedNumRecords;
+
+ this.parallelism = toCopy.parallelism;
+ this.minimalMemoryPerSubTask = toCopy.minimalMemoryPerSubTask;
+
+ this.id = toCopy.id;
+ this.costWeight = toCopy.costWeight;
+ this.onDynamicPath = toCopy.onDynamicPath;
+ }
+
+ // ------------------------------------------------------------------------
+ // Methods specific to unary- / binary- / special nodes
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the name of this node, which is the name of the function/operator, or
+ * data source / data sink.
+ *
+ * @return The node name.
+ */
+ public abstract String getName();
+
+ /**
+ * This function connects the predecessors to this operator.
+ *
+ * @param operatorToNode The map from program operators to optimizer nodes.
+ * @param defaultExchangeMode The data exchange mode to use, if the operator does not
+ * specify one.
+ */
+ public abstract void setInput(Map<Operator<?>, OptimizerNode> operatorToNode,
+ ExecutionMode defaultExchangeMode);
+
+ /**
+ * This function connects the operators that produce the broadcast inputs to this operator.
+ *
+ * @param operatorToNode The map from program operators to optimizer nodes.
+ * @param defaultExchangeMode The data exchange mode to use, if the operator does not
+ * specify one.
+ *
+ * @throws CompilerException
+ */
+ public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) {
+ // skip for Operators that don't support broadcast variables
+ if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) {
+ return;
+ }
+
+ // get all broadcast inputs
+ AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator());
+
+ // create connections and add them
+ for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) {
+ OptimizerNode predecessor = operatorToNode.get(input.getValue());
+ DagConnection connection = new DagConnection(predecessor, this,
+ ShipStrategyType.BROADCAST, defaultExchangeMode);
+ addBroadcastConnection(input.getKey(), connection);
+ predecessor.addOutgoingConnection(connection);
+ }
+ }
+
+ /**
+ * Gets all incoming connections of this node.
+ * This method needs to be overridden by subclasses to return the children.
+ *
+ * @return The list of incoming connections.
+ */
+ public abstract List<DagConnection> getIncomingConnections();
+
+ /**
+ * Tells the node to compute the interesting properties for its inputs. The interesting properties
+ * for the node itself must have been computed before.
+ * The node must then see how many of interesting properties it preserves and add its own.
+ *
+ * @param estimator The {@code CostEstimator} instance to use for plan cost estimation.
+ */
+ public abstract void computeInterestingPropertiesForInputs(CostEstimator estimator);
+
+ /**
+ * This method causes the node to compute the description of open branches in its sub-plan. An open branch
+ * describes, that a (transitive) child node had multiple outputs, which have not all been re-joined in the
+ * sub-plan. This method needs to set the <code>openBranches</code> field to a stack of unclosed branches, the
+ * latest one top. A branch is considered closed, if some later node sees all of the branching node's outputs,
+ * no matter if there have been more branches to different paths in the meantime.
+ */
+ public abstract void computeUnclosedBranchStack();
+
+
+ protected List<UnclosedBranchDescriptor> computeUnclosedBranchStackForBroadcastInputs(
+ List<UnclosedBranchDescriptor> branchesSoFar)
+ {
+ // handle the data flow branching for the broadcast inputs
+ for (DagConnection broadcastInput : getBroadcastConnections()) {
+ OptimizerNode bcSource = broadcastInput.getSource();
+ addClosedBranches(bcSource.closedBranchingNodes);
+
+ List<UnclosedBranchDescriptor> bcBranches = bcSource.getBranchesForParent(broadcastInput);
+
+ ArrayList<UnclosedBranchDescriptor> mergedBranches = new ArrayList<UnclosedBranchDescriptor>();
+ mergeLists(branchesSoFar, bcBranches, mergedBranches, true);
+ branchesSoFar = mergedBranches.isEmpty() ? Collections.<UnclosedBranchDescriptor>emptyList() : mergedBranches;
+ }
+
+ return branchesSoFar;
+ }
+
+ /**
+ * Computes the plan alternatives for this node, an implicitly for all nodes that are children of
+ * this node. This method must determine for each alternative the global and local properties
+ * and the costs. This method may recursively call <code>getAlternatives()</code> on its children
+ * to get their plan alternatives, and build its own alternatives on top of those.
+ *
+ * @param estimator
+ * The cost estimator used to estimate the costs of each plan alternative.
+ * @return A list containing all plan alternatives.
+ */
+ public abstract List<PlanNode> getAlternativePlans(CostEstimator estimator);
+
+ /**
+ * This method implements the visit of a depth-first graph traversing visitor. Implementers must first
+ * call the <code>preVisit()</code> method, then hand the visitor to their children, and finally call
+ * the <code>postVisit()</code> method.
+ *
+ * @param visitor
+ * The graph traversing visitor.
+ * @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
+ */
+ @Override
+ public abstract void accept(Visitor<OptimizerNode> visitor);
+
+ public abstract SemanticProperties getSemanticProperties();
+
+ // ------------------------------------------------------------------------
+ // Getters / Setters
+ // ------------------------------------------------------------------------
+
+ @Override
+ public Iterable<OptimizerNode> getPredecessors() {
+ List<OptimizerNode> allPredecessors = new ArrayList<OptimizerNode>();
+
+ for (DagConnection dagConnection : getIncomingConnections()) {
+ allPredecessors.add(dagConnection.getSource());
+ }
+
+ for (DagConnection conn : getBroadcastConnections()) {
+ allPredecessors.add(conn.getSource());
+ }
+
+ return allPredecessors;
+ }
+
+ /**
+ * Gets the ID of this node. If the id has not yet been set, this method returns -1;
+ *
+ * @return This node's id, or -1, if not yet set.
+ */
+ public int getId() {
+ return this.id;
+ }
+
+ /**
+ * Sets the ID of this node.
+ *
+ * @param id
+ * The id for this node.
+ */
+ public void initId(int id) {
+ if (id <= 0) {
+ throw new IllegalArgumentException();
+ }
+
+ if (this.id == -1) {
+ this.id = id;
+ } else {
+ throw new IllegalStateException("Id has already been initialized.");
+ }
+ }
+
+ /**
+ * Adds the broadcast connection identified by the given {@code name} to this node.
+ *
+ * @param broadcastConnection The connection to add.
+ */
+ public void addBroadcastConnection(String name, DagConnection broadcastConnection) {
+ this.broadcastConnectionNames.add(name);
+ this.broadcastConnections.add(broadcastConnection);
+ }
+
+ /**
+ * Return the list of names associated with broadcast inputs for this node.
+ */
+ public List<String> getBroadcastConnectionNames() {
+ return this.broadcastConnectionNames;
+ }
+
+ /**
+ * Return the list of inputs associated with broadcast variables for this node.
+ */
+ public List<DagConnection> getBroadcastConnections() {
+ return this.broadcastConnections;
+ }
+
+ /**
+ * Adds a new outgoing connection to this node.
+ *
+ * @param connection
+ * The connection to add.
+ */
+ public void addOutgoingConnection(DagConnection connection) {
+ if (this.outgoingConnections == null) {
+ this.outgoingConnections = new ArrayList<DagConnection>();
+ } else {
+ if (this.outgoingConnections.size() == 64) {
+ throw new CompilerException("Cannot currently handle nodes with more than 64 outputs.");
+ }
+ }
+
+ this.outgoingConnections.add(connection);
+ }
+
+ /**
+ * The list of outgoing connections from this node to succeeding tasks.
+ *
+ * @return The list of outgoing connections.
+ */
+ public List<DagConnection> getOutgoingConnections() {
+ return this.outgoingConnections;
+ }
+
+ /**
+ * Gets the operator represented by this optimizer node.
+ *
+ * @return This node's operator.
+ */
+ public Operator<?> getOperator() {
+ return this.operator;
+ }
+
+ /**
+ * Gets the parallelism for the operator represented by this optimizer node.
+ * The parallelism denotes how many parallel instances of the operator on will be
+ * spawned during the execution. If this value is <code>-1</code>, then the system will take
+ * the default number of parallel instances.
+ *
+ * @return The parallelism of the operator.
+ */
+ public int getParallelism() {
+ return this.parallelism;
+ }
+
+ /**
+ * Sets the parallelism for this optimizer node.
+ * The parallelism denotes how many parallel instances of the operator will be
+ * spawned during the execution. If this value is set to <code>-1</code>, then the system will take
+ * the default number of parallel instances.
+ *
+ * @param parallelism The parallelism to set.
+ * @throws IllegalArgumentException If the parallelism is smaller than one and not -1.
+ */
+ public void setDegreeOfParallelism(int parallelism) {
+ if (parallelism < 1 && parallelism != -1) {
+ throw new IllegalArgumentException("Degree of parallelism of " + parallelism + " is invalid.");
+ }
+ this.parallelism = parallelism;
+ }
+
+ /**
+ * Gets the amount of memory that all subtasks of this task have jointly available.
+ *
+ * @return The total amount of memory across all subtasks.
+ */
+ public long getMinimalMemoryAcrossAllSubTasks() {
+ return this.minimalMemoryPerSubTask == -1 ? -1 : this.minimalMemoryPerSubTask * this.parallelism;
+ }
+
+ public boolean isOnDynamicPath() {
+ return this.onDynamicPath;
+ }
+
+ public void identifyDynamicPath(int costWeight) {
+ boolean anyDynamic = false;
+ boolean allDynamic = true;
+
+ for (DagConnection conn : getIncomingConnections()) {
+ boolean dynamicIn = conn.isOnDynamicPath();
+ anyDynamic |= dynamicIn;
+ allDynamic &= dynamicIn;
+ }
+
+ for (DagConnection conn : getBroadcastConnections()) {
+ boolean dynamicIn = conn.isOnDynamicPath();
+ anyDynamic |= dynamicIn;
+ allDynamic &= dynamicIn;
+ }
+
+ if (anyDynamic) {
+ this.onDynamicPath = true;
+ this.costWeight = costWeight;
+ if (!allDynamic) {
+ // this node joins static and dynamic path.
+ // mark the connections where the source is not dynamic as cached
+ for (DagConnection conn : getIncomingConnections()) {
+ if (!conn.getSource().isOnDynamicPath()) {
+ conn.setMaterializationMode(conn.getMaterializationMode().makeCached());
+ }
+ }
+
+ // broadcast variables are always cached, because they stay unchanged available in the
+ // runtime context of the functions
+ }
+ }
+ }
+
+ public int getCostWeight() {
+ return this.costWeight;
+ }
+
+ public int getMaxDepth() {
+ int maxDepth = 0;
+ for (DagConnection conn : getIncomingConnections()) {
+ maxDepth = Math.max(maxDepth, conn.getMaxDepth());
+ }
+ for (DagConnection conn : getBroadcastConnections()) {
+ maxDepth = Math.max(maxDepth, conn.getMaxDepth());
+ }
+
+ return maxDepth;
+ }
+
+ /**
+ * Gets the properties that are interesting for this node to produce.
+ *
+ * @return The interesting properties for this node, or null, if not yet computed.
+ */
+ public InterestingProperties getInterestingProperties() {
+ return this.intProps;
+ }
+
+ @Override
+ public long getEstimatedOutputSize() {
+ return this.estimatedOutputSize;
+ }
+
+ @Override
+ public long getEstimatedNumRecords() {
+ return this.estimatedNumRecords;
+ }
+
+ public void setEstimatedOutputSize(long estimatedOutputSize) {
+ this.estimatedOutputSize = estimatedOutputSize;
+ }
+
+ public void setEstimatedNumRecords(long estimatedNumRecords) {
+ this.estimatedNumRecords = estimatedNumRecords;
+ }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() {
+ if (this.estimatedOutputSize > 0 && this.estimatedNumRecords > 0) {
+ return ((float) this.estimatedOutputSize) / this.estimatedNumRecords;
+ } else {
+ return -1.0f;
+ }
+ }
+
+ /**
+ * Checks whether this node has branching output. A node's output is branched, if it has more
+ * than one output connection.
+ *
+ * @return True, if the node's output branches. False otherwise.
+ */
+ public boolean isBranching() {
+ return getOutgoingConnections() != null && getOutgoingConnections().size() > 1;
+ }
+
+ public void markAllOutgoingConnectionsAsPipelineBreaking() {
+ if (this.outgoingConnections == null) {
+ throw new IllegalStateException("The outgoing connections have not yet been initialized.");
+ }
+ for (DagConnection conn : getOutgoingConnections()) {
+ conn.markBreaksPipeline();
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Miscellaneous
+ // ------------------------------------------------------------------------
+
+ /**
+ * Checks, if all outgoing connections have their interesting properties set from their target nodes.
+ *
+ * @return True, if on all outgoing connections, the interesting properties are set. False otherwise.
+ */
+ public boolean haveAllOutputConnectionInterestingProperties() {
+ for (DagConnection conn : getOutgoingConnections()) {
+ if (conn.getInterestingProperties() == null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Computes all the interesting properties that are relevant to this node. The interesting
+ * properties are a union of the interesting properties on each outgoing connection.
+ * However, if two interesting properties on the outgoing connections overlap,
+ * the interesting properties will occur only once in this set. For that, this
+ * method deduplicates and merges the interesting properties.
+ * This method returns copies of the original interesting properties objects and
+ * leaves the original objects, contained by the connections, unchanged.
+ */
+ public void computeUnionOfInterestingPropertiesFromSuccessors() {
+ List<DagConnection> conns = getOutgoingConnections();
+ if (conns.size() == 0) {
+ // no incoming, we have none ourselves
+ this.intProps = new InterestingProperties();
+ } else {
+ this.intProps = conns.get(0).getInterestingProperties().clone();
+ for (int i = 1; i < conns.size(); i++) {
+ this.intProps.addInterestingProperties(conns.get(i).getInterestingProperties());
+ }
+ }
+ this.intProps.dropTrivials();
+ }
+
+ public void clearInterestingProperties() {
+ this.intProps = null;
+ for (DagConnection conn : getIncomingConnections()) {
+ conn.clearInterestingProperties();
+ }
+ for (DagConnection conn : getBroadcastConnections()) {
+ conn.clearInterestingProperties();
+ }
+ }
+
+ /**
+ * Causes this node to compute its output estimates (such as number of rows, size in bytes)
+ * based on the inputs and the compiler hints. The compiler hints are instantiated with conservative
+ * default values which are used if no other values are provided. Nodes may access the statistics to
+ * determine relevant information.
+ *
+ * @param statistics
+ * The statistics object which may be accessed to get statistical information.
+ * The parameter may be null, if no statistics are available.
+ */
+ public void computeOutputEstimates(DataStatistics statistics) {
+ // sanity checking
+ for (DagConnection c : getIncomingConnections()) {
+ if (c.getSource() == null) {
+ throw new CompilerException("Bug: Estimate computation called before inputs have been set.");
+ }
+ }
+
+ // let every operator do its computation
+ computeOperatorSpecificDefaultEstimates(statistics);
+
+ if (this.estimatedOutputSize < 0) {
+ this.estimatedOutputSize = -1;
+ }
+ if (this.estimatedNumRecords < 0) {
+ this.estimatedNumRecords = -1;
+ }
+
+ // overwrite default estimates with hints, if given
+ if (getOperator() == null || getOperator().getCompilerHints() == null) {
+ return ;
+ }
+
+ CompilerHints hints = getOperator().getCompilerHints();
+ if (hints.getOutputSize() >= 0) {
+ this.estimatedOutputSize = hints.getOutputSize();
+ }
+
+ if (hints.getOutputCardinality() >= 0) {
+ this.estimatedNumRecords = hints.getOutputCardinality();
+ }
+
+ if (hints.getFilterFactor() >= 0.0f) {
+ if (this.estimatedNumRecords >= 0) {
+ this.estimatedNumRecords = (long) (this.estimatedNumRecords * hints.getFilterFactor());
+
+ if (this.estimatedOutputSize >= 0) {
+ this.estimatedOutputSize = (long) (this.estimatedOutputSize * hints.getFilterFactor());
+ }
+ }
+ else if (this instanceof SingleInputNode) {
+ OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode();
+ if (pred != null && pred.getEstimatedNumRecords() >= 0) {
+ this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor());
+ }
+ }
+ }
+
+ // use the width to infer the cardinality (given size) and vice versa
+ if (hints.getAvgOutputRecordSize() >= 1) {
+ // the estimated number of rows based on size
+ if (this.estimatedNumRecords == -1 && this.estimatedOutputSize >= 0) {
+ this.estimatedNumRecords = (long) (this.estimatedOutputSize / hints.getAvgOutputRecordSize());
+ }
+ else if (this.estimatedOutputSize == -1 && this.estimatedNumRecords >= 0) {
+ this.estimatedOutputSize = (long) (this.estimatedNumRecords * hints.getAvgOutputRecordSize());
+ }
+ }
+ }
+
+ protected abstract void computeOperatorSpecificDefaultEstimates(DataStatistics statistics);
+
+ // ------------------------------------------------------------------------
+ // Reading of stub annotations
+ // ------------------------------------------------------------------------
+
+ /**
+ * Reads all stub annotations, i.e. which fields remain constant, what cardinality bounds the
+ * functions have, which fields remain unique.
+ */
+ protected void readStubAnnotations() {
+ readUniqueFieldsAnnotation();
+ }
+
+ protected void readUniqueFieldsAnnotation() {
+ if (this.operator.getCompilerHints() != null) {
+ Set<FieldSet> uniqueFieldSets = operator.getCompilerHints().getUniqueFields();
+ if (uniqueFieldSets != null) {
+ if (this.uniqueFields == null) {
+ this.uniqueFields = new HashSet<FieldSet>();
+ }
+ this.uniqueFields.addAll(uniqueFieldSets);
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Access of stub annotations
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the FieldSets which are unique in the output of the node.
+ */
+ public Set<FieldSet> getUniqueFields() {
+ return this.uniqueFields == null ? Collections.<FieldSet>emptySet() : this.uniqueFields;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Pruning
+ // --------------------------------------------------------------------------------------------
+
+ protected void prunePlanAlternatives(List<PlanNode> plans) {
+ if (plans.isEmpty()) {
+ throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
+ }
+ // shortcut for the simple case
+ if (plans.size() == 1) {
+ return;
+ }
+
+ // we can only compare plan candidates that made equal choices
+ // at the branching points. for each choice at a branching point,
+ // we need to keep the cheapest (wrt. interesting properties).
+ // if we do not keep candidates for each branch choice, we might not
+ // find branch compatible candidates when joining the branches back.
+
+ // for pruning, we are quasi AFTER the node, so in the presence of
+ // branches, we need form the per-branch-choice groups by the choice
+ // they made at the latest un-joined branching node. Note that this is
+ // different from the check for branch compatibility of candidates, as
+ // this happens on the input sub-plans and hence BEFORE the node (therefore
+ // it is relevant to find the latest (partially) joined branch point.
+
+ if (this.openBranches == null || this.openBranches.isEmpty()) {
+ prunePlanAlternativesWithCommonBranching(plans);
+ } else {
+ // partition the candidates into groups that made the same sub-plan candidate
+ // choice at the latest unclosed branch point
+
+ final OptimizerNode[] branchDeterminers = new OptimizerNode[this.openBranches.size()];
+
+ for (int i = 0; i < branchDeterminers.length; i++) {
+ branchDeterminers[i] = this.openBranches.get(this.openBranches.size() - 1 - i).getBranchingNode();
+ }
+
+ // this sorter sorts by the candidate choice at the branch point
+ Comparator<PlanNode> sorter = new Comparator<PlanNode>() {
+
+ @Override
+ public int compare(PlanNode o1, PlanNode o2) {
+ for (OptimizerNode branchDeterminer : branchDeterminers) {
+ PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminer);
+ PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminer);
+ int hash1 = System.identityHashCode(n1);
+ int hash2 = System.identityHashCode(n2);
+
+ if (hash1 != hash2) {
+ return hash1 - hash2;
+ }
+ }
+ return 0;
+ }
+ };
+ Collections.sort(plans, sorter);
+
+ List<PlanNode> result = new ArrayList<PlanNode>();
+ List<PlanNode> turn = new ArrayList<PlanNode>();
+
+ final PlanNode[] determinerChoice = new PlanNode[branchDeterminers.length];
+
+ while (!plans.isEmpty()) {
+ // take one as the determiner
+ turn.clear();
+ PlanNode determiner = plans.remove(plans.size() - 1);
+ turn.add(determiner);
+
+ for (int i = 0; i < determinerChoice.length; i++) {
+ determinerChoice[i] = determiner.getCandidateAtBranchPoint(branchDeterminers[i]);
+ }
+
+ // go backwards through the plans and find all that are equal
+ boolean stillEqual = true;
+ for (int k = plans.size() - 1; k >= 0 && stillEqual; k--) {
+ PlanNode toCheck = plans.get(k);
+
+ for (int i = 0; i < branchDeterminers.length; i++) {
+ PlanNode checkerChoice = toCheck.getCandidateAtBranchPoint(branchDeterminers[i]);
+
+ if (checkerChoice != determinerChoice[i]) {
+ // not the same anymore
+ stillEqual = false;
+ break;
+ }
+ }
+
+ if (stillEqual) {
+ // the same
+ plans.remove(k);
+ turn.add(toCheck);
+ }
+ }
+
+ // now that we have only plans with the same branch alternatives, prune!
+ if (turn.size() > 1) {
+ prunePlanAlternativesWithCommonBranching(turn);
+ }
+ result.addAll(turn);
+ }
+
+ // after all turns are complete
+ plans.clear();
+ plans.addAll(result);
+ }
+ }
+
+ protected void prunePlanAlternativesWithCommonBranching(List<PlanNode> plans) {
+ // for each interesting property, which plans are cheapest
+ final RequestedGlobalProperties[] gps = this.intProps.getGlobalProperties().toArray(
+ new RequestedGlobalProperties[this.intProps.getGlobalProperties().size()]);
+ final RequestedLocalProperties[] lps = this.intProps.getLocalProperties().toArray(
+ new RequestedLocalProperties[this.intProps.getLocalProperties().size()]);
+
+ final PlanNode[][] toKeep = new PlanNode[gps.length][];
+ final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
+
+
+ PlanNode cheapest = null; // the overall cheapest plan
+
+ // go over all plans from the list
+ for (PlanNode candidate : plans) {
+ // check if that plan is the overall cheapest
+ if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
+ cheapest = candidate;
+ }
+
+ // find the interesting global properties that this plan matches
+ for (int i = 0; i < gps.length; i++) {
+ if (gps[i].isMetBy(candidate.getGlobalProperties())) {
+ // the candidate meets the global property requirements. That means
+ // it has a chance that its local properties are re-used (they would be
+ // destroyed if global properties need to be established)
+
+ if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
+ cheapestForGlobal[i] = candidate;
+ }
+
+ final PlanNode[] localMatches;
+ if (toKeep[i] == null) {
+ localMatches = new PlanNode[lps.length];
+ toKeep[i] = localMatches;
+ } else {
+ localMatches = toKeep[i];
+ }
+
+ for (int k = 0; k < lps.length; k++) {
+ if (lps[k].isMetBy(candidate.getLocalProperties())) {
+ final PlanNode previous = localMatches[k];
+ if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) {
+ // this one is cheaper!
+ localMatches[k] = candidate;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // all plans are set now
+ plans.clear();
+
+ // add the cheapest plan
+ if (cheapest != null) {
+ plans.add(cheapest);
+ cheapest.setPruningMarker(); // remember that that plan is in the set
+ }
+
+ // add all others, which are optimal for some interesting properties
+ for (int i = 0; i < gps.length; i++) {
+ if (toKeep[i] != null) {
+ final PlanNode[] localMatches = toKeep[i];
+ for (final PlanNode n : localMatches) {
+ if (n != null && !n.isPruneMarkerSet()) {
+ n.setPruningMarker();
+ plans.add(n);
+ }
+ }
+ }
+ if (cheapestForGlobal[i] != null) {
+ final PlanNode n = cheapestForGlobal[i];
+ if (!n.isPruneMarkerSet()) {
+ n.setPruningMarker();
+ plans.add(n);
+ }
+ }
+ }
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // Handling of branches
+ // --------------------------------------------------------------------------------------------
+
+ public boolean hasUnclosedBranches() {
+ return this.openBranches != null && !this.openBranches.isEmpty();
+ }
+
+ public Set<OptimizerNode> getClosedBranchingNodes() {
+ return this.closedBranchingNodes;
+ }
+
+ public List<UnclosedBranchDescriptor> getOpenBranches() {
+ return this.openBranches;
+ }
+
+
+ protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection toParent) {
+ if (this.outgoingConnections.size() == 1) {
+ // return our own stack of open branches, because nothing is added
+ if (this.openBranches == null || this.openBranches.isEmpty()) {
+ return Collections.emptyList();
+ } else {
+ return new ArrayList<UnclosedBranchDescriptor>(this.openBranches);
+ }
+ }
+ else if (this.outgoingConnections.size() > 1) {
+ // we branch add a branch info to the stack
+ List<UnclosedBranchDescriptor> branches = new ArrayList<UnclosedBranchDescriptor>(4);
+ if (this.openBranches != null) {
+ branches.addAll(this.openBranches);
+ }
+
+ // find out, which output number the connection to the parent
+ int num;
+ for (num = 0; num < this.outgoingConnections.size(); num++) {
+ if (this.outgoingConnections.get(num) == toParent) {
+ break;
+ }
+ }
+ if (num >= this.outgoingConnections.size()) {
+ throw new CompilerException("Error in compiler: "
+ + "Parent to get branch info for is not contained in the outgoing connections.");
+ }
+
+ // create the description and add it
+ long bitvector = 0x1L << num;
+ branches.add(new UnclosedBranchDescriptor(this, bitvector));
+ return branches;
+ }
+ else {
+ throw new CompilerException(
+ "Error in compiler: Cannot get branch info for successor in a node with no successors.");
+ }
+ }
+
+
+ protected void removeClosedBranches(List<UnclosedBranchDescriptor> openList) {
+ if (openList == null || openList.isEmpty() || this.closedBranchingNodes == null || this.closedBranchingNodes.isEmpty()) {
+ return;
+ }
+
+ Iterator<UnclosedBranchDescriptor> it = openList.iterator();
+ while (it.hasNext()) {
+ if (this.closedBranchingNodes.contains(it.next().getBranchingNode())) {
+ //this branch was already closed --> remove it from the list
+ it.remove();
+ }
+ }
+ }
+
+ protected void addClosedBranches(Set<OptimizerNode> alreadyClosed) {
+ if (alreadyClosed == null || alreadyClosed.isEmpty()) {
+ return;
+ }
+
+ if (this.closedBranchingNodes == null) {
+ this.closedBranchingNodes = new HashSet<OptimizerNode>(alreadyClosed);
+ } else {
+ this.closedBranchingNodes.addAll(alreadyClosed);
+ }
+ }
+
+ protected void addClosedBranch(OptimizerNode alreadyClosed) {
+ if (this.closedBranchingNodes == null) {
+ this.closedBranchingNodes = new HashSet<OptimizerNode>();
+ }
+
+ this.closedBranchingNodes.add(alreadyClosed);
+ }
+
+ /**
+ * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
+ * alternative plans are comparable, if
+ *
+ * a) There is no branch in the sub-plan of this node
+ * b) Both candidates have the same candidate as the child at the last open branch.
+ *
+ * @param plan1 The root node of the first candidate plan.
+ * @param plan2 The root node of the second candidate plan.
+ * @return True if the nodes are branch compatible in the inputs.
+ */
+ protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
+ if (plan1 == null || plan2 == null) {
+ throw new NullPointerException();
+ }
+
+ // if there is no open branch, the children are always compatible.
+ // in most plans, that will be the dominant case
+ if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
+ return true;
+ }
+
+ for (OptimizerNode joinedBrancher : hereJoinedBranches) {
+ final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
+ final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
+
+ if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * The node IDs are assigned in graph-traversal order (pre-order), hence, each list is
+ * sorted by ID in ascending order and all consecutive lists start with IDs in ascending order.
+ *
+ * @param markJoinedBranchesAsPipelineBreaking True, if the
+ */
+ protected final boolean mergeLists(List<UnclosedBranchDescriptor> child1open,
+ List<UnclosedBranchDescriptor> child2open,
+ List<UnclosedBranchDescriptor> result,
+ boolean markJoinedBranchesAsPipelineBreaking) {
+
+ //remove branches which have already been closed
+ removeClosedBranches(child1open);
+ removeClosedBranches(child2open);
+
+ result.clear();
+
+ // check how many open branches we have. the cases:
+ // 1) if both are null or empty, the result is null
+ // 2) if one side is null (or empty), the result is the other side.
+ // 3) both are set, then we need to merge.
+ if (child1open == null || child1open.isEmpty()) {
+ if(child2open != null && !child2open.isEmpty()) {
+ result.addAll(child2open);
+ }
+ return false;
+ }
+
+ if (child2open == null || child2open.isEmpty()) {
+ result.addAll(child1open);
+ return false;
+ }
+
+ int index1 = child1open.size() - 1;
+ int index2 = child2open.size() - 1;
+
+ boolean didCloseABranch = false;
+
+ // as both lists (child1open and child2open) are sorted in ascending ID order
+ // we can do a merge-join-like loop which preserved the order in the result list
+ // and eliminates duplicates
+ while (index1 >= 0 || index2 >= 0) {
+ int id1 = -1;
+ int id2 = index2 >= 0 ? child2open.get(index2).getBranchingNode().getId() : -1;
+
+ while (index1 >= 0 && (id1 = child1open.get(index1).getBranchingNode().getId()) > id2) {
+ result.add(child1open.get(index1));
+ index1--;
+ }
+ while (index2 >= 0 && (id2 = child2open.get(index2).getBranchingNode().getId()) > id1) {
+ result.add(child2open.get(index2));
+ index2--;
+ }
+
+ // match: they share a common branching child
+ if (id1 == id2) {
+ didCloseABranch = true;
+
+ // if this is the latest common child, remember it
+ OptimizerNode currBanchingNode = child1open.get(index1).getBranchingNode();
+
+ long vector1 = child1open.get(index1).getJoinedPathsVector();
+ long vector2 = child2open.get(index2).getJoinedPathsVector();
+
+ // check if this is the same descriptor, (meaning that it contains the same paths)
+ // if it is the same, add it only once, otherwise process the join of the paths
+ if (vector1 == vector2) {
+ result.add(child1open.get(index1));
+ }
+ else {
+ // we merge (re-join) a branch
+
+ // mark the branch as a point where we break the pipeline
+ if (markJoinedBranchesAsPipelineBreaking) {
+ currBanchingNode.markAllOutgoingConnectionsAsPipelineBreaking();
+ }
+
+ if (this.hereJoinedBranches == null) {
+ this.hereJoinedBranches = new ArrayList<OptimizerNode>(2);
+ }
+ this.hereJoinedBranches.add(currBanchingNode);
+
+ // see, if this node closes the branch
+ long joinedInputs = vector1 | vector2;
+
+ // this is 2^size - 1, which is all bits set at positions 0..size-1
+ long allInputs = (0x1L << currBanchingNode.getOutgoingConnections().size()) - 1;
+
+ if (joinedInputs == allInputs) {
+ // closed - we can remove it from the stack
+ addClosedBranch(currBanchingNode);
+ } else {
+ // not quite closed
+ result.add(new UnclosedBranchDescriptor(currBanchingNode, joinedInputs));
+ }
+ }
+
+ index1--;
+ index2--;
+ }
+ }
+
+ // merged. now we need to reverse the list, because we added the elements in reverse order
+ Collections.reverse(result);
+ return didCloseABranch;
+ }
+
+ @Override
+ public OptimizerNode getOptimizerNode() {
+ return this;
+ }
+
+ @Override
+ public PlanNode getPlanNode() {
+ return null;
+ }
+
+ @Override
+ public Iterable<DumpableConnection<OptimizerNode>> getDumpableInputs() {
+ List<DumpableConnection<OptimizerNode>> allInputs = new ArrayList<DumpableConnection<OptimizerNode>>();
+
+ allInputs.addAll(getIncomingConnections());
+ allInputs.addAll(getBroadcastConnections());
+
+ return allInputs;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder bld = new StringBuilder();
+
+ bld.append(getName());
+ bld.append(" (").append(getOperator().getName()).append(") ");
+
+ int i = 1;
+ for (DagConnection conn : getIncomingConnections()) {
+ String shipStrategyName = conn.getShipStrategy() == null ? "null" : conn.getShipStrategy().name();
+ bld.append('(').append(i++).append(":").append(shipStrategyName).append(')');
+ }
+
+ return bld.toString();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Description of an unclosed branch. An unclosed branch is when the data flow branched (one operator's
+ * result is consumed by multiple targets), but these different branches (targets) have not been joined
+ * together.
+ */
+ public static final class UnclosedBranchDescriptor {
+
+ protected OptimizerNode branchingNode;
+
+ protected long joinedPathsVector;
+
+ /**
+ * Creates a new branching descriptor.
+ *
+ * @param branchingNode The node where the branch occurred (teh node with multiple outputs).
+ * @param joinedPathsVector A bit vector describing which branches are tracked by this descriptor.
+ * The bit vector is one, where the branch is tracked, zero otherwise.
+ */
+ protected UnclosedBranchDescriptor(OptimizerNode branchingNode, long joinedPathsVector) {
+ this.branchingNode = branchingNode;
+ this.joinedPathsVector = joinedPathsVector;
+ }
+
+ public OptimizerNode getBranchingNode() {
+ return this.branchingNode;
+ }
+
+ public long getJoinedPathsVector() {
+ return this.joinedPathsVector;
+ }
+
+ @Override
+ public String toString() {
+ return "(" + this.branchingNode.getOperator() + ") [" + this.joinedPathsVector + "]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
new file mode 100644
index 0000000..5c811b0
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ * The optimizer's internal representation of a <i>Partition</i> operator node.
+ */
+public class PartitionNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ public PartitionNode(PartitionOperatorBase<?> operator) {
+ super(operator);
+
+ OperatorDescriptorSingle descr = new PartitionDescriptor(
+ this.getOperator().getPartitionMethod(), this.keys, operator.getCustomPartitioner());
+ this.possibleProperties = Collections.singletonList(descr);
+ }
+
+ @Override
+ public PartitionOperatorBase<?> getOperator() {
+ return (PartitionOperatorBase<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Partition";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // partitioning does not change the number of records
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static class PartitionDescriptor extends OperatorDescriptorSingle {
+
+ private final PartitionMethod pMethod;
+ private final Partitioner<?> customPartitioner;
+
+ public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Partitioner<?> customPartitioner) {
+ super(pKeys);
+
+ this.pMethod = pMethod;
+ this.customPartitioner = customPartitioner;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.UNARY_NO_OP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Partition", in, DriverStrategy.UNARY_NO_OP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgps = new RequestedGlobalProperties();
+
+ switch (this.pMethod) {
+ case HASH:
+ rgps.setHashPartitioned(this.keys);
+ break;
+ case REBALANCE:
+ rgps.setForceRebalancing();
+ break;
+ case CUSTOM:
+ rgps.setCustomPartitioned(this.keys, this.customPartitioner);
+ break;
+ case RANGE:
+ throw new UnsupportedOperationException("Not yet supported");
+ default:
+ throw new IllegalArgumentException("Invalid partition method");
+ }
+
+ return Collections.singletonList(rgps);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ // partitioning does not require any local property.
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ // the partition node is a no-operation operation, such that all global properties are preserved.
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ // the partition node is a no-operation operation, such that all global properties are preserved.
+ return lProps;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
new file mode 100644
index 0000000..bbe4607
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.util.Visitor;
+
+final class PlanCacheCleaner implements Visitor<OptimizerNode> {
+
+ static final PlanCacheCleaner INSTANCE = new PlanCacheCleaner();
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ if (visitable.cachedPlans != null && visitable.isOnDynamicPath()) {
+ visitable.cachedPlans = null;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
new file mode 100644
index 0000000..1477038
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.AllReduceProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.optimizer.operators.ReduceProperties;
+
+/**
+ * The Optimizer representation of a <i>Reduce</i> operator.
+ */
+public class ReduceNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ private ReduceNode preReduceUtilityNode;
+
+
+ public ReduceNode(ReduceOperatorBase<?, ?> operator) {
+ super(operator);
+
+ if (this.keys == null) {
+ // case of a key-less reducer. force a parallelism of 1
+ setDegreeOfParallelism(1);
+ }
+
+ OperatorDescriptorSingle props = this.keys == null ?
+ new AllReduceProperties() :
+ new ReduceProperties(this.keys, operator.getCustomPartitioner());
+
+ this.possibleProperties = Collections.singletonList(props);
+ }
+
+ public ReduceNode(ReduceNode reducerToCopyForCombiner) {
+ super(reducerToCopyForCombiner);
+
+ this.possibleProperties = Collections.emptyList();
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public ReduceOperatorBase<?, ?> getOperator() {
+ return (ReduceOperatorBase<?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Reduce";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Estimates
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // no real estimates possible for a reducer.
+ }
+
+ public ReduceNode getCombinerUtilityNode() {
+ if (this.preReduceUtilityNode == null) {
+ this.preReduceUtilityNode = new ReduceNode(this);
+
+ // we conservatively assume the combiner returns the same data size as it consumes
+ this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ }
+ return this.preReduceUtilityNode;
+ }
+}
[07/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
new file mode 100644
index 0000000..8c19462
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/FeedbackPropertiesMatchTest.java
@@ -0,0 +1,1436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer;
+
+import static org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport.*;
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.operators.BinaryOperatorInformation;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.OperatorInformation;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.UnaryOperatorInformation;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dag.MapNode;
+import org.apache.flink.optimizer.dag.JoinNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.junit.Test;
+import org.apache.flink.api.java.io.TextInputFormat;
+
+
+public class FeedbackPropertiesMatchTest {
+
+ @Test
+ public void testNoPartialSolutionFoundSingleInputOnly() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
+
+ SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
+
+ Channel toMap1 = new Channel(target);
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(map1);
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = new LocalProperties();
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
+ assertTrue(report == NO_PARTIAL_SOLUTION);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSingleInputOperators() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
+
+ Channel toMap1 = new Channel(target);
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(map1);
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ // no feedback properties and none are ever required and present
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = new LocalProperties();
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some global feedback properties and none are ever required and present
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = new LocalProperties();
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some local feedback properties and none are ever required and present
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some global and local feedback properties and none are ever required and present
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // --------------------------- requirements on channel 1 -----------------------
+
+ // some required global properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = new LocalProperties();
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldList(2, 5));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required local properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1, 2));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global and local properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldList(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1, 2));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global and local properties, which are over-fulfilled
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldSet(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global properties that are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 1));
+ LocalProperties lp = new LocalProperties();
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldList(2, 5));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required local properties that are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required global and local properties where the global properties are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 1));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required global and local properties where the local properties are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // --------------------------- requirements on channel 2 -----------------------
+
+ // some required global properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = new LocalProperties();
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldList(2, 5));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required local properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1, 2));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global and local properties, which are matched exactly
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 5));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldList(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1, 2));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global and local properties, which are over-fulfilled
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1, 2));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldSet(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global properties that are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 1));
+ LocalProperties lp = new LocalProperties();
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setHashPartitioned(new FieldSet(2, 5));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required local properties that are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required global and local properties where the global properties are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(2, 1));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldSet(2, 5));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some required global and local properties where the local properties are not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // ---------------------- requirements mixed on 1 and 2 -----------------------
+
+ // some required global properties at step one and some more at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ RequestedGlobalProperties reqGp1 = new RequestedGlobalProperties();
+ reqGp1.setAnyPartitioning(new FieldList(1, 2));
+
+ RequestedGlobalProperties reqGp2 = new RequestedGlobalProperties();
+ reqGp2.setHashPartitioned(new FieldList(1, 2));
+
+ toMap1.setRequiredGlobalProps(reqGp1);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp2);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required local properties at step one and some more at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
+
+ RequestedLocalProperties reqLp1 = new RequestedLocalProperties();
+ reqLp1.setGroupedFields(new FieldList(3, 1));
+
+ RequestedLocalProperties reqLp2 = new RequestedLocalProperties();
+ reqLp2.setOrdering(new Ordering(3, null, Order.ANY).appendOrdering(1, null, Order.ANY));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp1);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp2);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required global properties at step one and some local ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some required local properties at step one and some global ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some fulfilled global properties at step one and some non-fulfilled local ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 3));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some fulfilled local properties at step one and some non-fulfilled global ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(2, 3));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some non-fulfilled global properties at step one and some fulfilled local ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(2, 3));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // some non-fulfilled local properties at step one and some fulfilled global ones at step 2
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldList(1, 2));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(2, 1, 3));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSingleInputOperatorsWithReCreation() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
+
+ Channel toMap1 = new Channel(target);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(map1);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ // set ship strategy in first channel, so later non matching global properties do not matter
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldSet(2, 5));
+
+ toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(reqGp);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(MET, report);
+ }
+
+ // set ship strategy in second channel, so previous non matching global properties void the match
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldSet(2, 5));
+
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap2.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 5), DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // set local strategy in first channel, so later non matching local properties do not matter
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(4, 1));
+
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // set local strategy in second channel, so previous non matching local properties void the match
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(4, 1));
+
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
+
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // create the properties on the same node as the requirement
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(1, 2));
+ LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldSet(5, 7));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(5, 7));
+
+ toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(5, 7), DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap1.setRequiredGlobalProps(reqGp);
+ toMap1.setRequiredLocalProps(reqLp);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(MET, report);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSingleInputOperatorsChainOfThree() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
+
+ Channel toMap1 = new Channel(target);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(map1);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ Channel toMap3 = new Channel(map2);
+ SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP);
+
+ // set local strategy in first channel, so later non matching local properties do not matter
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.forOrdering(new Ordering(3, null, Order.ASCENDING).appendOrdering(1, null, Order.DESCENDING));
+
+ RequestedLocalProperties reqLp = new RequestedLocalProperties();
+ reqLp.setGroupedFields(new FieldList(4, 1));
+
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false});
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap3.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ toMap3.setRequiredGlobalProps(null);
+ toMap3.setRequiredLocalProps(reqLp);
+
+ FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // set global strategy in first channel, so later non matching global properties do not matter
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(5, 3));
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ RequestedGlobalProperties reqGp = new RequestedGlobalProperties();
+ reqGp.setAnyPartitioning(new FieldSet(2, 3));
+
+ toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(1, 2), DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap3.setLocalStrategy(LocalStrategy.NONE);
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toMap2.setRequiredGlobalProps(null);
+ toMap2.setRequiredLocalProps(null);
+
+ toMap3.setRequiredGlobalProps(reqGp);
+ toMap3.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = map3.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testNoPartialSolutionFoundTwoInputOperator() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
+
+ SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
+ SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
+
+ Channel toMap1 = new Channel(source1);
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(source2);
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ Channel toJoin1 = new Channel(map1);
+ Channel toJoin2 = new Channel(map2);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin2.setLocalStrategy(LocalStrategy.NONE);
+
+ DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
+ assertEquals(NO_PARTIAL_SOLUTION, report);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTwoOperatorsOneIndependent() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
+ SourcePlanNode source = new SourcePlanNode(getSourceNode(), "Other Source");
+
+ Channel toMap1 = new Channel(target);
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(source);
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ Channel toJoin1 = new Channel(map1);
+ Channel toJoin2 = new Channel(map2);
+
+ DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
+
+ Channel toAfterJoin = new Channel(join);
+ toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
+
+ // attach some properties to the non-relevant input
+ {
+ toMap2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] {true, true});
+
+ RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
+ joinGp.setFullyReplicated();
+
+ RequestedLocalProperties joinLp = new RequestedLocalProperties();
+ joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
+
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin2.setLocalStrategy(LocalStrategy.NONE);
+ toJoin2.setRequiredGlobalProps(joinGp);
+ toJoin2.setRequiredLocalProps(joinLp);
+ }
+
+ // ------------------------------------------------------------------------------------
+
+ // no properties from the partial solution, no required properties
+ {
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some properties from the partial solution, no required properties
+ {
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+
+ // produced properties match relevant input
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2));
+
+ toJoin1.setRequiredGlobalProps(rgp);
+ toJoin1.setRequiredLocalProps(rlp);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // produced properties do not match relevant input
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(1, 2, 3));
+
+ toJoin1.setRequiredGlobalProps(rgp);
+ toJoin1.setRequiredLocalProps(rlp);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // produced properties overridden before join
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(rgp);
+ toMap1.setRequiredLocalProps(rlp);
+
+ toJoin1.setRequiredGlobalProps(null);
+ toJoin1.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(MET, report);
+ }
+
+ // produced properties before join match, after join match as well
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2, 1));
+
+ toMap1.setRequiredGlobalProps(null);
+ toMap1.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ toJoin1.setRequiredGlobalProps(rgp);
+ toJoin1.setRequiredLocalProps(rlp);
+
+ toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // produced properties before join match, after join do not match
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
+ rgp1.setHashPartitioned(new FieldList(0));
+
+ RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
+ rgp2.setHashPartitioned(new FieldList(3));
+
+ RequestedLocalProperties rlp1 = new RequestedLocalProperties();
+ rlp1.setGroupedFields(new FieldList(2, 1));
+
+ RequestedLocalProperties rlp2 = new RequestedLocalProperties();
+ rlp2.setGroupedFields(new FieldList(3, 4));
+
+ toJoin1.setRequiredGlobalProps(rgp1);
+ toJoin1.setRequiredLocalProps(rlp1);
+
+ toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
+
+ toAfterJoin.setRequiredGlobalProps(rgp2);
+ toAfterJoin.setRequiredLocalProps(rlp2);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // produced properties are overridden, does not matter that they do not match
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setAnyPartitioning(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(1));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(1, 2, 3));
+
+ toJoin1.setRequiredGlobalProps(null);
+ toJoin1.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1), DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(MET, report);
+ }
+
+ // local property overridden before join, local property mismatch after join not relevant
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setAnyPartitioning(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(1, 2, 3));
+
+ toJoin1.setRequiredGlobalProps(null);
+ toJoin1.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
+
+ toAfterJoin.setRequiredGlobalProps(null);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // local property overridden before join, global property mismatch after join void the match
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setAnyPartitioning(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(1));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(1, 2, 3));
+
+ toJoin1.setRequiredGlobalProps(null);
+ toJoin1.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTwoOperatorsBothDependent() {
+ try {
+ SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
+
+ Channel toMap1 = new Channel(target);
+ toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap1.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
+
+ Channel toMap2 = new Channel(target);
+ toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toMap2.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
+
+ Channel toJoin1 = new Channel(map1);
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ Channel toJoin2 = new Channel(map2);
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin2.setLocalStrategy(LocalStrategy.NONE);
+
+ DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
+
+ Channel toAfterJoin = new Channel(join);
+ toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
+ SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
+
+ // no properties from the partial solution, no required properties
+ {
+ GlobalProperties gp = new GlobalProperties();
+ LocalProperties lp = LocalProperties.EMPTY;
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // some properties from the partial solution, no required properties
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // test requirements on one input and met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2, 1));
+
+ toJoin1.setRequiredGlobalProps(rgp);
+ toJoin1.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // test requirements on both input and met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2, 1));
+
+ toJoin1.setRequiredGlobalProps(rgp);
+ toJoin1.setRequiredLocalProps(rlp);
+
+ toJoin2.setRequiredGlobalProps(rgp);
+ toJoin2.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
+ }
+
+ // test requirements on both inputs, one not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
+ rgp1.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp1 = new RequestedLocalProperties();
+ rlp1.setGroupedFields(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
+ rgp2.setHashPartitioned(new FieldList(1));
+
+ RequestedLocalProperties rlp2 = new RequestedLocalProperties();
+ rlp2.setGroupedFields(new FieldList(0, 3));
+
+ toJoin1.setRequiredGlobalProps(rgp1);
+ toJoin1.setRequiredLocalProps(rlp1);
+
+ toJoin2.setRequiredGlobalProps(rgp2);
+ toJoin2.setRequiredLocalProps(rlp2);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // test override on both inputs, later requirement ignored
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(1));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(0, 3));
+
+ toJoin1.setRequiredGlobalProps(null);
+ toJoin1.setRequiredLocalProps(null);
+
+ toJoin2.setRequiredGlobalProps(null);
+ toJoin2.setRequiredLocalProps(null);
+
+ toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
+ toJoin2.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED);
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(MET, report);
+ }
+
+ // test override on one inputs, later requirement met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(0));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(2, 1));
+
+ toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(PENDING, report);
+ }
+
+ // test override on one input, later requirement not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(3));
+
+ RequestedLocalProperties rlp = new RequestedLocalProperties();
+ rlp.setGroupedFields(new FieldList(77, 69));
+
+ toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88), DataExchangeMode.PIPELINED);
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(rlp);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+
+ // test override on one input locally, later global requirement not met
+ {
+ GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(new FieldList(0));
+ LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
+
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setHashPartitioned(new FieldList(3));
+
+
+ toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(3), new boolean[] { false });
+
+ toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+ toJoin1.setLocalStrategy(LocalStrategy.NONE);
+
+ toAfterJoin.setRequiredGlobalProps(rgp);
+ toAfterJoin.setRequiredLocalProps(null);
+
+ FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
+ assertEquals(NOT_MET, report);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static DataSourceNode getSourceNode() {
+ return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>(new TextInputFormat(new Path("/")), new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO)));
+ }
+
+ private static MapNode getMapNode() {
+ return new MapNode(new MapOperatorBase<String, String, MapFunction<String,String>>(new IdentityMapper<String>(), new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), "map op"));
+ }
+
+ private static JoinNode getJoinNode() {
+ return new JoinNode(new JoinOperatorBase<String, String, String, FlatJoinFunction<String, String, String>>(new DummyFlatJoinFunction<String>(), new BinaryOperatorInformation<String, String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[] {1}, new int[] {2}, "join op"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
new file mode 100644
index 0000000..77d185d
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/GroupOrderTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.record.operators.CoGroupOperator;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.util.DummyCoGroupStub;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.types.DoubleValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.StringValue;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This test case has been created to validate that correct strategies are used if orders within groups are
+ * requested.
+ */
+@SuppressWarnings({"serial", "deprecation"})
+public class GroupOrderTest extends CompilerTestBase {
+
+ @Test
+ public void testReduceWithGroupOrder() {
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+
+ ReduceOperator reduce = ReduceOperator.builder(new IdentityReduce()).keyField(IntValue.class, 2).name("Reduce").input(source).build();
+ Ordering groupOrder = new Ordering(5, StringValue.class, Order.DESCENDING);
+ reduce.setGroupOrder(groupOrder);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, reduce, "Sink");
+
+
+ Plan plan = new Plan(sink, "Test Temp Task");
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+ OptimizedPlan oPlan;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ return; // silence the compiler
+ }
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ SinkPlanNode sinkNode = resolver.getNode("Sink");
+ SingleInputPlanNode reducer = resolver.getNode("Reduce");
+
+ // verify the strategies
+ Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
+ Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
+
+ Channel c = reducer.getInput();
+ Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
+
+ FieldList ship = new FieldList(2);
+ FieldList local = new FieldList(2, 5);
+ Assert.assertEquals(ship, c.getShipStrategyKeys());
+ Assert.assertEquals(local, c.getLocalStrategyKeys());
+ Assert.assertTrue(c.getLocalStrategySortOrder()[0] == reducer.getSortOrders(0)[0]);
+
+ // check that we indeed sort descending
+ Assert.assertTrue(c.getLocalStrategySortOrder()[1] == groupOrder.getFieldSortDirections()[0]);
+ }
+
+ @Test
+ public void testCoGroupWithGroupOrder() {
+ // construct the plan
+ FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source1");
+ FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source2");
+
+ CoGroupOperator coGroup = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 3, 6)
+ .keyField(LongValue.class, 0, 0)
+ .name("CoGroup").input1(source1).input2(source2).build();
+
+ Ordering groupOrder1 = new Ordering(5, StringValue.class, Order.DESCENDING);
+ Ordering groupOrder2 = new Ordering(1, StringValue.class, Order.DESCENDING);
+ groupOrder2.appendOrdering(4, DoubleValue.class, Order.ASCENDING);
+ coGroup.setGroupOrderForInputOne(groupOrder1);
+ coGroup.setGroupOrderForInputTwo(groupOrder2);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, coGroup, "Sink");
+
+ Plan plan = new Plan(sink, "Reduce Group Order Test");
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+ OptimizedPlan oPlan;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ return; // silence the compiler
+ }
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ SinkPlanNode sinkNode = resolver.getNode("Sink");
+ DualInputPlanNode coGroupNode = resolver.getNode("CoGroup");
+
+ // verify the strategies
+ Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
+ Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy());
+ Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy());
+
+ Channel c1 = coGroupNode.getInput1();
+ Channel c2 = coGroupNode.getInput2();
+
+ Assert.assertEquals(LocalStrategy.SORT, c1.getLocalStrategy());
+ Assert.assertEquals(LocalStrategy.SORT, c2.getLocalStrategy());
+
+ FieldList ship1 = new FieldList(new int[] {3, 0});
+ FieldList ship2 = new FieldList(new int[] {6, 0});
+
+ FieldList local1 = new FieldList(new int[] {3, 0, 5});
+ FieldList local2 = new FieldList(new int[] {6, 0, 1, 4});
+
+ Assert.assertEquals(ship1, c1.getShipStrategyKeys());
+ Assert.assertEquals(ship2, c2.getShipStrategyKeys());
+ Assert.assertEquals(local1, c1.getLocalStrategyKeys());
+ Assert.assertEquals(local2, c2.getLocalStrategyKeys());
+
+ Assert.assertTrue(c1.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
+ Assert.assertTrue(c1.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
+ Assert.assertTrue(c2.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
+ Assert.assertTrue(c2.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
+
+ // check that the local group orderings are correct
+ Assert.assertTrue(c1.getLocalStrategySortOrder()[2] == groupOrder1.getFieldSortDirections()[0]);
+ Assert.assertTrue(c2.getLocalStrategySortOrder()[2] == groupOrder2.getFieldSortDirections()[0]);
+ Assert.assertTrue(c2.getLocalStrategySortOrder()[3] == groupOrder2.getFieldSortDirections()[1]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
new file mode 100644
index 0000000..6dadc19
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/HardPlansCompilationTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.record.operators.CrossOperator;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.DummyCrossStub;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityMap;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.apache.flink.types.IntValue;
+import org.junit.Test;
+
+/**
+ * This class tests plans that once failed because of a bug:
+ * <ul>
+ * <li> Ticket 158
+ * </ul>
+ */
+@SuppressWarnings({"serial", "deprecation"})
+public class HardPlansCompilationTest extends CompilerTestBase {
+
+ /**
+ * Source -> Map -> Reduce -> Cross -> Reduce -> Cross -> Reduce ->
+ * |--------------------------/ /
+ * |--------------------------------------------/
+ *
+ * First cross has SameKeyFirst output contract
+ */
+ @Test
+ public void testTicket158() {
+ // construct the plan
+ FileDataSource source = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source");
+
+ MapOperator map = MapOperator.builder(new IdentityMap()).name("Map1").input(source).build();
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce1").input(map).build();
+
+ CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub()).name("Cross1").input1(reduce1).input2(source).build();
+
+ ReduceOperator reduce2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce2").input(cross1).build();
+
+ CrossOperator cross2 = CrossOperator.builder(new DummyCrossStub()).name("Cross2").input1(reduce2).input2(source).build();
+
+ ReduceOperator reduce3 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0).name("Reduce3").input(cross2).build();
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, "Sink");
+ sink.setInput(reduce3);
+
+ Plan plan = new Plan(sink, "Test Temp Task");
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+ jobGen.compileJobGraph(oPlan);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
new file mode 100644
index 0000000..ac4f820
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/IterationsCompilerTest.java
@@ -0,0 +1,409 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.java.aggregation.Aggregations;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichJoinFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+
+
+@SuppressWarnings({"serial", "unchecked"})
+public class IterationsCompilerTest extends CompilerTestBase {
+
+ @Test
+ public void testSolutionSetDeltaDependsOnBroadcastVariable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> source =
+ env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
+
+ DataSet<Tuple2<Long, Long>> invariantInput =
+ env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
+
+ // iteration from here
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = source.iterateDelta(source, 1000, 1);
+
+ DataSet<Tuple2<Long, Long>> result =
+ invariantInput
+ .map(new IdentityMapper<Tuple2<Long, Long>>()).withBroadcastSet(iter.getWorkset(), "bc data")
+ .join(iter.getSolutionSet()).where(0).equalTo(1).projectFirst(1).projectSecond(1);
+
+ iter.closeWith(result.map(new IdentityMapper<Tuple2<Long,Long>>()), result).print();
+
+ OptimizedPlan p = compileNoStats(env.createProgramPlan());
+
+ // check that the JSON generator accepts this plan
+ new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(p);
+
+ // check that the JobGraphGenerator accepts the plan
+ new JobGraphGenerator().compileJobGraph(p);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTwoIterationsWithMapperInbetween() throws Exception {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> bulkResult = doBulkIteration(verticesWithInitialId, edges);
+
+ DataSet<Tuple2<Long, Long>> mappedBulk = bulkResult.map(new DummyMap());
+
+ DataSet<Tuple2<Long, Long>> depResult = doDeltaIteration(mappedBulk, edges);
+
+ depResult.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(1, op.getDataSinks().size());
+ assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
+
+ WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
+ assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTwoIterationsDirectlyChained() throws Exception {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> bulkResult = doBulkIteration(verticesWithInitialId, edges);
+
+ DataSet<Tuple2<Long, Long>> depResult = doDeltaIteration(bulkResult, edges);
+
+ depResult.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(1, op.getDataSinks().size());
+ assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
+
+ WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, wipn.getInput1().getShipStrategy());
+ assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testTwoWorksetIterationsDirectlyChained() throws Exception {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
+ DataSet<Tuple2<Long, Long>> firstResult = doDeltaIteration(verticesWithInitialId, edges);
+
+ DataSet<Tuple2<Long, Long>> secondResult = doDeltaIteration(firstResult, edges);
+
+ secondResult.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(1, op.getDataSinks().size());
+ assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
+
+ WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy());
+ assertTrue(wipn.getInput2().getTempMode().breaksPipeline());
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIterationPushingWorkOut() throws Exception {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<Long, Long>> input1 = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
+
+ DataSet<Tuple2<Long, Long>> input2 = env.readCsvFile("/some/file/path").types(Long.class, Long.class);
+
+ doBulkIteration(input1, input2).print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ assertEquals(1, op.getDataSinks().size());
+ assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof BulkIterationPlanNode);
+
+ BulkIterationPlanNode bipn = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
+
+ // check that work has not! been pushed out, as the end of the step function does not produce the necessary properties
+ for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) {
+ assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy());
+ }
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testWorksetIterationPipelineBreakerPlacement() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ // the workset (input two of the delta iteration) is the same as what is consumed be the successive join
+ DataSet<Tuple2<Long, Long>> initialWorkset = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
+
+ DataSet<Tuple2<Long, Long>> initialSolutionSet = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue());
+
+ // trivial iteration, since we are interested in the inputs to the iteration
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = initialSolutionSet.iterateDelta(initialWorkset, 100, 0);
+
+ DataSet<Tuple2<Long, Long>> next = iteration.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>());
+
+ DataSet<Tuple2<Long, Long>> result = iteration.closeWith(next, next);
+
+ initialWorkset
+ .join(result, JoinHint.REPARTITION_HASH_FIRST)
+ .where(0).equalTo(0)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ compileNoStats(p);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testResetPartialSolution() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> width = env.generateSequence(1, 10);
+ DataSet<Long> update = env.generateSequence(1, 10);
+ DataSet<Long> lastGradient = env.generateSequence(1, 10);
+
+ DataSet<Long> init = width.union(update).union(lastGradient);
+
+ IterativeDataSet<Long> iteration = init.iterate(10);
+
+ width = iteration.filter(new IdFilter<Long>());
+ update = iteration.filter(new IdFilter<Long>());
+ lastGradient = iteration.filter(new IdFilter<Long>());
+
+ DataSet<Long> gradient = width.map(new IdentityMapper<Long>());
+ DataSet<Long> term = gradient.join(lastGradient)
+ .where(new IdentityKeyExtractor<Long>())
+ .equalTo(new IdentityKeyExtractor<Long>())
+ .with(new JoinFunction<Long, Long, Long>() {
+ public Long join(Long first, Long second) { return null; }
+ });
+
+ update = update.map(new RichMapFunction<Long, Long>() {
+ public Long map(Long value) { return null; }
+ }).withBroadcastSet(term, "some-name");
+
+ DataSet<Long> result = iteration.closeWith(width.union(update).union(lastGradient));
+
+ result.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static DataSet<Tuple2<Long, Long>> doBulkIteration(DataSet<Tuple2<Long, Long>> vertices, DataSet<Tuple2<Long, Long>> edges) {
+
+ // open a bulk iteration
+ IterativeDataSet<Tuple2<Long, Long>> iteration = vertices.iterate(20);
+
+ DataSet<Tuple2<Long, Long>> changes = iteration
+ .join(edges).where(0).equalTo(0).with(new Join222())
+ .groupBy(0).aggregate(Aggregations.MIN, 1)
+ .join(iteration).where(0).equalTo(0)
+ .flatMap(new FlatMapJoin());
+
+ // close the bulk iteration
+ return iteration.closeWith(changes);
+ }
+
+
+ public static DataSet<Tuple2<Long, Long>> doDeltaIteration(DataSet<Tuple2<Long, Long>> vertices, DataSet<Tuple2<Long, Long>> edges) {
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> depIteration = vertices.iterateDelta(vertices, 100, 0);
+
+ DataSet<Tuple1<Long>> candidates = depIteration.getWorkset().join(edges).where(0).equalTo(0)
+ .projectSecond(1);
+
+ DataSet<Tuple1<Long>> grouped = candidates.groupBy(0).reduceGroup(new Reduce101());
+
+ DataSet<Tuple2<Long, Long>> candidatesDependencies =
+ grouped.join(edges).where(0).equalTo(1).projectSecond(0, 1);
+
+ DataSet<Tuple2<Long, Long>> verticesWithNewComponents =
+ candidatesDependencies.join(depIteration.getSolutionSet()).where(0).equalTo(0)
+ .with(new Join222())
+ .groupBy(0).aggregate(Aggregations.MIN, 1);
+
+ DataSet<Tuple2<Long, Long>> updatedComponentId =
+ verticesWithNewComponents.join(depIteration.getSolutionSet()).where(0).equalTo(0)
+ .flatMap(new FlatMapJoin());
+
+ DataSet<Tuple2<Long, Long>> depResult = depIteration.closeWith(updatedComponentId, updatedComponentId);
+
+ return depResult;
+
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static final class Join222 extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ @Override
+ public Tuple2<Long, Long> join(Tuple2<Long, Long> vertexWithComponent, Tuple2<Long, Long> edge) {
+ return null;
+ }
+ }
+
+ public static final class FlatMapJoin extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
+
+ @Override
+ public void flatMap(Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> value, Collector<Tuple2<Long, Long>> out) {}
+ }
+
+ public static final class DummyMap extends RichMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+
+ @Override
+ public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception {
+ return value;
+ }
+ }
+
+ @ForwardedFields("0")
+ public static final class Reduce101 extends RichGroupReduceFunction<Tuple1<Long>, Tuple1<Long>> {
+
+ @Override
+ public void reduce(Iterable<Tuple1<Long>> values, Collector<Tuple1<Long>> out) {}
+ }
+
+ @ForwardedFields("0")
+ public static final class DuplicateValue extends RichMapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
+
+ @Override
+ public Tuple2<Long, Long> map(Tuple1<Long> value) throws Exception {
+ return new Tuple2<Long, Long>(value.f0, value.f0);
+ }
+ }
+
+ public static final class DuplicateValueScalar<T> extends RichMapFunction<T, Tuple2<T, T>> {
+
+ @Override
+ public Tuple2<T, T> map(T value) {
+ return new Tuple2<T, T>(value, value);
+ }
+ }
+
+ public static final class IdFilter<T> implements FilterFunction<T> {
+ @Override
+ public boolean filter(T value) {
+ return true;
+ }
+ }
+}
[37/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
deleted file mode 100644
index 0cad34e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/OptimizerNode.java
+++ /dev/null
@@ -1,1172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.AbstractUdfOperator;
-import org.apache.flink.api.common.operators.CompilerHints;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plandump.DumpableConnection;
-import org.apache.flink.optimizer.plandump.DumpableNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitable;
-import org.apache.flink.util.Visitor;
-
-/**
- * The OptimizerNode is the base class of all nodes in the optimizer DAG. The optimizer DAG is the
- * optimizer's representation of a program, created before the actual optimization (which creates different
- * candidate plans and computes their cost).
- * <p>>
- * Nodes in the DAG correspond (almost) one-to-one to the operators in a program. The optimizer DAG is constructed
- * to hold the additional information that the optimizer needs:
- * <ul>
- * <li>Estimates of the data size processed by each operator</li>
- * <li>Helper structures to track where the data flow "splits" and "joins", to support flows that are
- * DAGs but not trees.</li>
- * <li>Tags and weights to differentiate between loop-variant and -invariant parts of an iteration</li>
- * <li>Interesting properties to be used during the enumeration of candidate plans</li>
- * </ul>
- */
-public abstract class OptimizerNode implements Visitable<OptimizerNode>, EstimateProvider, DumpableNode<OptimizerNode> {
-
- public static final int MAX_DYNAMIC_PATH_COST_WEIGHT = 100;
-
- // --------------------------------------------------------------------------------------------
- // Members
- // --------------------------------------------------------------------------------------------
-
- private final Operator<?> operator; // The operator (Reduce / Join / DataSource / ...)
-
- private List<String> broadcastConnectionNames = new ArrayList<String>(); // the broadcast inputs names of this node
-
- private List<DagConnection> broadcastConnections = new ArrayList<DagConnection>(); // the broadcast inputs of this node
-
- private List<DagConnection> outgoingConnections; // The links to succeeding nodes
-
- private InterestingProperties intProps; // the interesting properties of this node
-
- // --------------------------------- Branch Handling ------------------------------------------
-
- protected List<UnclosedBranchDescriptor> openBranches; // stack of branches in the sub-graph that are not joined
-
- protected Set<OptimizerNode> closedBranchingNodes; // stack of branching nodes which have already been closed
-
- protected List<OptimizerNode> hereJoinedBranches; // the branching nodes (node with multiple outputs)
- // that are partially joined (through multiple inputs or broadcast vars)
-
- // ---------------------------- Estimates and Annotations -------------------------------------
-
- protected long estimatedOutputSize = -1; // the estimated size of the output (bytes)
-
- protected long estimatedNumRecords = -1; // the estimated number of key/value pairs in the output
-
- protected Set<FieldSet> uniqueFields; // set of attributes that will always be unique after this node
-
- // --------------------------------- General Parameters ---------------------------------------
-
- private int parallelism = -1; // the number of parallel instances of this node
-
- private long minimalMemoryPerSubTask = -1;
-
- protected int id = -1; // the id for this node.
-
- protected int costWeight = 1; // factor to weight the costs for dynamic paths
-
- protected boolean onDynamicPath;
-
- protected List<PlanNode> cachedPlans; // cache candidates, because the may be accessed repeatedly
-
- // ------------------------------------------------------------------------
- // Constructor / Setup
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new optimizer node that represents the given program operator.
- *
- * @param op The operator that the node represents.
- */
- public OptimizerNode(Operator<?> op) {
- this.operator = op;
- readStubAnnotations();
- }
-
- protected OptimizerNode(OptimizerNode toCopy) {
- this.operator = toCopy.operator;
- this.intProps = toCopy.intProps;
-
- this.openBranches = toCopy.openBranches;
- this.closedBranchingNodes = toCopy.closedBranchingNodes;
-
- this.estimatedOutputSize = toCopy.estimatedOutputSize;
- this.estimatedNumRecords = toCopy.estimatedNumRecords;
-
- this.parallelism = toCopy.parallelism;
- this.minimalMemoryPerSubTask = toCopy.minimalMemoryPerSubTask;
-
- this.id = toCopy.id;
- this.costWeight = toCopy.costWeight;
- this.onDynamicPath = toCopy.onDynamicPath;
- }
-
- // ------------------------------------------------------------------------
- // Methods specific to unary- / binary- / special nodes
- // ------------------------------------------------------------------------
-
- /**
- * Gets the name of this node, which is the name of the function/operator, or
- * data source / data sink.
- *
- * @return The node name.
- */
- public abstract String getName();
-
- /**
- * This function connects the predecessors to this operator.
- *
- * @param operatorToNode The map from program operators to optimizer nodes.
- * @param defaultExchangeMode The data exchange mode to use, if the operator does not
- * specify one.
- */
- public abstract void setInput(Map<Operator<?>, OptimizerNode> operatorToNode,
- ExecutionMode defaultExchangeMode);
-
- /**
- * This function connects the operators that produce the broadcast inputs to this operator.
- *
- * @param operatorToNode The map from program operators to optimizer nodes.
- * @param defaultExchangeMode The data exchange mode to use, if the operator does not
- * specify one.
- *
- * @throws CompilerException
- */
- public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) {
- // skip for Operators that don't support broadcast variables
- if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) {
- return;
- }
-
- // get all broadcast inputs
- AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator());
-
- // create connections and add them
- for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) {
- OptimizerNode predecessor = operatorToNode.get(input.getValue());
- DagConnection connection = new DagConnection(predecessor, this,
- ShipStrategyType.BROADCAST, defaultExchangeMode);
- addBroadcastConnection(input.getKey(), connection);
- predecessor.addOutgoingConnection(connection);
- }
- }
-
- /**
- * Gets all incoming connections of this node.
- * This method needs to be overridden by subclasses to return the children.
- *
- * @return The list of incoming connections.
- */
- public abstract List<DagConnection> getIncomingConnections();
-
- /**
- * Tells the node to compute the interesting properties for its inputs. The interesting properties
- * for the node itself must have been computed before.
- * The node must then see how many of interesting properties it preserves and add its own.
- *
- * @param estimator The {@code CostEstimator} instance to use for plan cost estimation.
- */
- public abstract void computeInterestingPropertiesForInputs(CostEstimator estimator);
-
- /**
- * This method causes the node to compute the description of open branches in its sub-plan. An open branch
- * describes, that a (transitive) child node had multiple outputs, which have not all been re-joined in the
- * sub-plan. This method needs to set the <code>openBranches</code> field to a stack of unclosed branches, the
- * latest one top. A branch is considered closed, if some later node sees all of the branching node's outputs,
- * no matter if there have been more branches to different paths in the meantime.
- */
- public abstract void computeUnclosedBranchStack();
-
-
- protected List<UnclosedBranchDescriptor> computeUnclosedBranchStackForBroadcastInputs(
- List<UnclosedBranchDescriptor> branchesSoFar)
- {
- // handle the data flow branching for the broadcast inputs
- for (DagConnection broadcastInput : getBroadcastConnections()) {
- OptimizerNode bcSource = broadcastInput.getSource();
- addClosedBranches(bcSource.closedBranchingNodes);
-
- List<UnclosedBranchDescriptor> bcBranches = bcSource.getBranchesForParent(broadcastInput);
-
- ArrayList<UnclosedBranchDescriptor> mergedBranches = new ArrayList<UnclosedBranchDescriptor>();
- mergeLists(branchesSoFar, bcBranches, mergedBranches, true);
- branchesSoFar = mergedBranches.isEmpty() ? Collections.<UnclosedBranchDescriptor>emptyList() : mergedBranches;
- }
-
- return branchesSoFar;
- }
-
- /**
- * Computes the plan alternatives for this node, an implicitly for all nodes that are children of
- * this node. This method must determine for each alternative the global and local properties
- * and the costs. This method may recursively call <code>getAlternatives()</code> on its children
- * to get their plan alternatives, and build its own alternatives on top of those.
- *
- * @param estimator
- * The cost estimator used to estimate the costs of each plan alternative.
- * @return A list containing all plan alternatives.
- */
- public abstract List<PlanNode> getAlternativePlans(CostEstimator estimator);
-
- /**
- * This method implements the visit of a depth-first graph traversing visitor. Implementers must first
- * call the <code>preVisit()</code> method, then hand the visitor to their children, and finally call
- * the <code>postVisit()</code> method.
- *
- * @param visitor
- * The graph traversing visitor.
- * @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
- */
- @Override
- public abstract void accept(Visitor<OptimizerNode> visitor);
-
- public abstract SemanticProperties getSemanticProperties();
-
- // ------------------------------------------------------------------------
- // Getters / Setters
- // ------------------------------------------------------------------------
-
- @Override
- public Iterable<OptimizerNode> getPredecessors() {
- List<OptimizerNode> allPredecessors = new ArrayList<OptimizerNode>();
-
- for (DagConnection dagConnection : getIncomingConnections()) {
- allPredecessors.add(dagConnection.getSource());
- }
-
- for (DagConnection conn : getBroadcastConnections()) {
- allPredecessors.add(conn.getSource());
- }
-
- return allPredecessors;
- }
-
- /**
- * Gets the ID of this node. If the id has not yet been set, this method returns -1;
- *
- * @return This node's id, or -1, if not yet set.
- */
- public int getId() {
- return this.id;
- }
-
- /**
- * Sets the ID of this node.
- *
- * @param id
- * The id for this node.
- */
- public void initId(int id) {
- if (id <= 0) {
- throw new IllegalArgumentException();
- }
-
- if (this.id == -1) {
- this.id = id;
- } else {
- throw new IllegalStateException("Id has already been initialized.");
- }
- }
-
- /**
- * Adds the broadcast connection identified by the given {@code name} to this node.
- *
- * @param broadcastConnection The connection to add.
- */
- public void addBroadcastConnection(String name, DagConnection broadcastConnection) {
- this.broadcastConnectionNames.add(name);
- this.broadcastConnections.add(broadcastConnection);
- }
-
- /**
- * Return the list of names associated with broadcast inputs for this node.
- */
- public List<String> getBroadcastConnectionNames() {
- return this.broadcastConnectionNames;
- }
-
- /**
- * Return the list of inputs associated with broadcast variables for this node.
- */
- public List<DagConnection> getBroadcastConnections() {
- return this.broadcastConnections;
- }
-
- /**
- * Adds a new outgoing connection to this node.
- *
- * @param connection
- * The connection to add.
- */
- public void addOutgoingConnection(DagConnection connection) {
- if (this.outgoingConnections == null) {
- this.outgoingConnections = new ArrayList<DagConnection>();
- } else {
- if (this.outgoingConnections.size() == 64) {
- throw new CompilerException("Cannot currently handle nodes with more than 64 outputs.");
- }
- }
-
- this.outgoingConnections.add(connection);
- }
-
- /**
- * The list of outgoing connections from this node to succeeding tasks.
- *
- * @return The list of outgoing connections.
- */
- public List<DagConnection> getOutgoingConnections() {
- return this.outgoingConnections;
- }
-
- /**
- * Gets the operator represented by this optimizer node.
- *
- * @return This node's operator.
- */
- public Operator<?> getOperator() {
- return this.operator;
- }
-
- /**
- * Gets the parallelism for the operator represented by this optimizer node.
- * The parallelism denotes how many parallel instances of the operator on will be
- * spawned during the execution. If this value is <code>-1</code>, then the system will take
- * the default number of parallel instances.
- *
- * @return The parallelism of the operator.
- */
- public int getParallelism() {
- return this.parallelism;
- }
-
- /**
- * Sets the parallelism for this optimizer node.
- * The parallelism denotes how many parallel instances of the operator will be
- * spawned during the execution. If this value is set to <code>-1</code>, then the system will take
- * the default number of parallel instances.
- *
- * @param parallelism The parallelism to set.
- * @throws IllegalArgumentException If the parallelism is smaller than one and not -1.
- */
- public void setDegreeOfParallelism(int parallelism) {
- if (parallelism < 1 && parallelism != -1) {
- throw new IllegalArgumentException("Degree of parallelism of " + parallelism + " is invalid.");
- }
- this.parallelism = parallelism;
- }
-
- /**
- * Gets the amount of memory that all subtasks of this task have jointly available.
- *
- * @return The total amount of memory across all subtasks.
- */
- public long getMinimalMemoryAcrossAllSubTasks() {
- return this.minimalMemoryPerSubTask == -1 ? -1 : this.minimalMemoryPerSubTask * this.parallelism;
- }
-
- public boolean isOnDynamicPath() {
- return this.onDynamicPath;
- }
-
- public void identifyDynamicPath(int costWeight) {
- boolean anyDynamic = false;
- boolean allDynamic = true;
-
- for (DagConnection conn : getIncomingConnections()) {
- boolean dynamicIn = conn.isOnDynamicPath();
- anyDynamic |= dynamicIn;
- allDynamic &= dynamicIn;
- }
-
- for (DagConnection conn : getBroadcastConnections()) {
- boolean dynamicIn = conn.isOnDynamicPath();
- anyDynamic |= dynamicIn;
- allDynamic &= dynamicIn;
- }
-
- if (anyDynamic) {
- this.onDynamicPath = true;
- this.costWeight = costWeight;
- if (!allDynamic) {
- // this node joins static and dynamic path.
- // mark the connections where the source is not dynamic as cached
- for (DagConnection conn : getIncomingConnections()) {
- if (!conn.getSource().isOnDynamicPath()) {
- conn.setMaterializationMode(conn.getMaterializationMode().makeCached());
- }
- }
-
- // broadcast variables are always cached, because they stay unchanged available in the
- // runtime context of the functions
- }
- }
- }
-
- public int getCostWeight() {
- return this.costWeight;
- }
-
- public int getMaxDepth() {
- int maxDepth = 0;
- for (DagConnection conn : getIncomingConnections()) {
- maxDepth = Math.max(maxDepth, conn.getMaxDepth());
- }
- for (DagConnection conn : getBroadcastConnections()) {
- maxDepth = Math.max(maxDepth, conn.getMaxDepth());
- }
-
- return maxDepth;
- }
-
- /**
- * Gets the properties that are interesting for this node to produce.
- *
- * @return The interesting properties for this node, or null, if not yet computed.
- */
- public InterestingProperties getInterestingProperties() {
- return this.intProps;
- }
-
- @Override
- public long getEstimatedOutputSize() {
- return this.estimatedOutputSize;
- }
-
- @Override
- public long getEstimatedNumRecords() {
- return this.estimatedNumRecords;
- }
-
- public void setEstimatedOutputSize(long estimatedOutputSize) {
- this.estimatedOutputSize = estimatedOutputSize;
- }
-
- public void setEstimatedNumRecords(long estimatedNumRecords) {
- this.estimatedNumRecords = estimatedNumRecords;
- }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() {
- if (this.estimatedOutputSize > 0 && this.estimatedNumRecords > 0) {
- return ((float) this.estimatedOutputSize) / this.estimatedNumRecords;
- } else {
- return -1.0f;
- }
- }
-
- /**
- * Checks whether this node has branching output. A node's output is branched, if it has more
- * than one output connection.
- *
- * @return True, if the node's output branches. False otherwise.
- */
- public boolean isBranching() {
- return getOutgoingConnections() != null && getOutgoingConnections().size() > 1;
- }
-
- public void markAllOutgoingConnectionsAsPipelineBreaking() {
- if (this.outgoingConnections == null) {
- throw new IllegalStateException("The outgoing connections have not yet been initialized.");
- }
- for (DagConnection conn : getOutgoingConnections()) {
- conn.markBreaksPipeline();
- }
- }
-
- // ------------------------------------------------------------------------
- // Miscellaneous
- // ------------------------------------------------------------------------
-
- /**
- * Checks, if all outgoing connections have their interesting properties set from their target nodes.
- *
- * @return True, if on all outgoing connections, the interesting properties are set. False otherwise.
- */
- public boolean haveAllOutputConnectionInterestingProperties() {
- for (DagConnection conn : getOutgoingConnections()) {
- if (conn.getInterestingProperties() == null) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Computes all the interesting properties that are relevant to this node. The interesting
- * properties are a union of the interesting properties on each outgoing connection.
- * However, if two interesting properties on the outgoing connections overlap,
- * the interesting properties will occur only once in this set. For that, this
- * method deduplicates and merges the interesting properties.
- * This method returns copies of the original interesting properties objects and
- * leaves the original objects, contained by the connections, unchanged.
- */
- public void computeUnionOfInterestingPropertiesFromSuccessors() {
- List<DagConnection> conns = getOutgoingConnections();
- if (conns.size() == 0) {
- // no incoming, we have none ourselves
- this.intProps = new InterestingProperties();
- } else {
- this.intProps = conns.get(0).getInterestingProperties().clone();
- for (int i = 1; i < conns.size(); i++) {
- this.intProps.addInterestingProperties(conns.get(i).getInterestingProperties());
- }
- }
- this.intProps.dropTrivials();
- }
-
- public void clearInterestingProperties() {
- this.intProps = null;
- for (DagConnection conn : getIncomingConnections()) {
- conn.clearInterestingProperties();
- }
- for (DagConnection conn : getBroadcastConnections()) {
- conn.clearInterestingProperties();
- }
- }
-
- /**
- * Causes this node to compute its output estimates (such as number of rows, size in bytes)
- * based on the inputs and the compiler hints. The compiler hints are instantiated with conservative
- * default values which are used if no other values are provided. Nodes may access the statistics to
- * determine relevant information.
- *
- * @param statistics
- * The statistics object which may be accessed to get statistical information.
- * The parameter may be null, if no statistics are available.
- */
- public void computeOutputEstimates(DataStatistics statistics) {
- // sanity checking
- for (DagConnection c : getIncomingConnections()) {
- if (c.getSource() == null) {
- throw new CompilerException("Bug: Estimate computation called before inputs have been set.");
- }
- }
-
- // let every operator do its computation
- computeOperatorSpecificDefaultEstimates(statistics);
-
- if (this.estimatedOutputSize < 0) {
- this.estimatedOutputSize = -1;
- }
- if (this.estimatedNumRecords < 0) {
- this.estimatedNumRecords = -1;
- }
-
- // overwrite default estimates with hints, if given
- if (getOperator() == null || getOperator().getCompilerHints() == null) {
- return ;
- }
-
- CompilerHints hints = getOperator().getCompilerHints();
- if (hints.getOutputSize() >= 0) {
- this.estimatedOutputSize = hints.getOutputSize();
- }
-
- if (hints.getOutputCardinality() >= 0) {
- this.estimatedNumRecords = hints.getOutputCardinality();
- }
-
- if (hints.getFilterFactor() >= 0.0f) {
- if (this.estimatedNumRecords >= 0) {
- this.estimatedNumRecords = (long) (this.estimatedNumRecords * hints.getFilterFactor());
-
- if (this.estimatedOutputSize >= 0) {
- this.estimatedOutputSize = (long) (this.estimatedOutputSize * hints.getFilterFactor());
- }
- }
- else if (this instanceof SingleInputNode) {
- OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode();
- if (pred != null && pred.getEstimatedNumRecords() >= 0) {
- this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor());
- }
- }
- }
-
- // use the width to infer the cardinality (given size) and vice versa
- if (hints.getAvgOutputRecordSize() >= 1) {
- // the estimated number of rows based on size
- if (this.estimatedNumRecords == -1 && this.estimatedOutputSize >= 0) {
- this.estimatedNumRecords = (long) (this.estimatedOutputSize / hints.getAvgOutputRecordSize());
- }
- else if (this.estimatedOutputSize == -1 && this.estimatedNumRecords >= 0) {
- this.estimatedOutputSize = (long) (this.estimatedNumRecords * hints.getAvgOutputRecordSize());
- }
- }
- }
-
- protected abstract void computeOperatorSpecificDefaultEstimates(DataStatistics statistics);
-
- // ------------------------------------------------------------------------
- // Reading of stub annotations
- // ------------------------------------------------------------------------
-
- /**
- * Reads all stub annotations, i.e. which fields remain constant, what cardinality bounds the
- * functions have, which fields remain unique.
- */
- protected void readStubAnnotations() {
- readUniqueFieldsAnnotation();
- }
-
- protected void readUniqueFieldsAnnotation() {
- if (this.operator.getCompilerHints() != null) {
- Set<FieldSet> uniqueFieldSets = operator.getCompilerHints().getUniqueFields();
- if (uniqueFieldSets != null) {
- if (this.uniqueFields == null) {
- this.uniqueFields = new HashSet<FieldSet>();
- }
- this.uniqueFields.addAll(uniqueFieldSets);
- }
- }
- }
-
- // ------------------------------------------------------------------------
- // Access of stub annotations
- // ------------------------------------------------------------------------
-
- /**
- * Gets the FieldSets which are unique in the output of the node.
- */
- public Set<FieldSet> getUniqueFields() {
- return this.uniqueFields == null ? Collections.<FieldSet>emptySet() : this.uniqueFields;
- }
-
- // --------------------------------------------------------------------------------------------
- // Pruning
- // --------------------------------------------------------------------------------------------
-
- protected void prunePlanAlternatives(List<PlanNode> plans) {
- if (plans.isEmpty()) {
- throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
- }
- // shortcut for the simple case
- if (plans.size() == 1) {
- return;
- }
-
- // we can only compare plan candidates that made equal choices
- // at the branching points. for each choice at a branching point,
- // we need to keep the cheapest (wrt. interesting properties).
- // if we do not keep candidates for each branch choice, we might not
- // find branch compatible candidates when joining the branches back.
-
- // for pruning, we are quasi AFTER the node, so in the presence of
- // branches, we need form the per-branch-choice groups by the choice
- // they made at the latest un-joined branching node. Note that this is
- // different from the check for branch compatibility of candidates, as
- // this happens on the input sub-plans and hence BEFORE the node (therefore
- // it is relevant to find the latest (partially) joined branch point.
-
- if (this.openBranches == null || this.openBranches.isEmpty()) {
- prunePlanAlternativesWithCommonBranching(plans);
- } else {
- // partition the candidates into groups that made the same sub-plan candidate
- // choice at the latest unclosed branch point
-
- final OptimizerNode[] branchDeterminers = new OptimizerNode[this.openBranches.size()];
-
- for (int i = 0; i < branchDeterminers.length; i++) {
- branchDeterminers[i] = this.openBranches.get(this.openBranches.size() - 1 - i).getBranchingNode();
- }
-
- // this sorter sorts by the candidate choice at the branch point
- Comparator<PlanNode> sorter = new Comparator<PlanNode>() {
-
- @Override
- public int compare(PlanNode o1, PlanNode o2) {
- for (OptimizerNode branchDeterminer : branchDeterminers) {
- PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminer);
- PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminer);
- int hash1 = System.identityHashCode(n1);
- int hash2 = System.identityHashCode(n2);
-
- if (hash1 != hash2) {
- return hash1 - hash2;
- }
- }
- return 0;
- }
- };
- Collections.sort(plans, sorter);
-
- List<PlanNode> result = new ArrayList<PlanNode>();
- List<PlanNode> turn = new ArrayList<PlanNode>();
-
- final PlanNode[] determinerChoice = new PlanNode[branchDeterminers.length];
-
- while (!plans.isEmpty()) {
- // take one as the determiner
- turn.clear();
- PlanNode determiner = plans.remove(plans.size() - 1);
- turn.add(determiner);
-
- for (int i = 0; i < determinerChoice.length; i++) {
- determinerChoice[i] = determiner.getCandidateAtBranchPoint(branchDeterminers[i]);
- }
-
- // go backwards through the plans and find all that are equal
- boolean stillEqual = true;
- for (int k = plans.size() - 1; k >= 0 && stillEqual; k--) {
- PlanNode toCheck = plans.get(k);
-
- for (int i = 0; i < branchDeterminers.length; i++) {
- PlanNode checkerChoice = toCheck.getCandidateAtBranchPoint(branchDeterminers[i]);
-
- if (checkerChoice != determinerChoice[i]) {
- // not the same anymore
- stillEqual = false;
- break;
- }
- }
-
- if (stillEqual) {
- // the same
- plans.remove(k);
- turn.add(toCheck);
- }
- }
-
- // now that we have only plans with the same branch alternatives, prune!
- if (turn.size() > 1) {
- prunePlanAlternativesWithCommonBranching(turn);
- }
- result.addAll(turn);
- }
-
- // after all turns are complete
- plans.clear();
- plans.addAll(result);
- }
- }
-
- protected void prunePlanAlternativesWithCommonBranching(List<PlanNode> plans) {
- // for each interesting property, which plans are cheapest
- final RequestedGlobalProperties[] gps = this.intProps.getGlobalProperties().toArray(
- new RequestedGlobalProperties[this.intProps.getGlobalProperties().size()]);
- final RequestedLocalProperties[] lps = this.intProps.getLocalProperties().toArray(
- new RequestedLocalProperties[this.intProps.getLocalProperties().size()]);
-
- final PlanNode[][] toKeep = new PlanNode[gps.length][];
- final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
-
-
- PlanNode cheapest = null; // the overall cheapest plan
-
- // go over all plans from the list
- for (PlanNode candidate : plans) {
- // check if that plan is the overall cheapest
- if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
- cheapest = candidate;
- }
-
- // find the interesting global properties that this plan matches
- for (int i = 0; i < gps.length; i++) {
- if (gps[i].isMetBy(candidate.getGlobalProperties())) {
- // the candidate meets the global property requirements. That means
- // it has a chance that its local properties are re-used (they would be
- // destroyed if global properties need to be established)
-
- if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
- cheapestForGlobal[i] = candidate;
- }
-
- final PlanNode[] localMatches;
- if (toKeep[i] == null) {
- localMatches = new PlanNode[lps.length];
- toKeep[i] = localMatches;
- } else {
- localMatches = toKeep[i];
- }
-
- for (int k = 0; k < lps.length; k++) {
- if (lps[k].isMetBy(candidate.getLocalProperties())) {
- final PlanNode previous = localMatches[k];
- if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) {
- // this one is cheaper!
- localMatches[k] = candidate;
- }
- }
- }
- }
- }
- }
-
- // all plans are set now
- plans.clear();
-
- // add the cheapest plan
- if (cheapest != null) {
- plans.add(cheapest);
- cheapest.setPruningMarker(); // remember that that plan is in the set
- }
-
- // add all others, which are optimal for some interesting properties
- for (int i = 0; i < gps.length; i++) {
- if (toKeep[i] != null) {
- final PlanNode[] localMatches = toKeep[i];
- for (final PlanNode n : localMatches) {
- if (n != null && !n.isPruneMarkerSet()) {
- n.setPruningMarker();
- plans.add(n);
- }
- }
- }
- if (cheapestForGlobal[i] != null) {
- final PlanNode n = cheapestForGlobal[i];
- if (!n.isPruneMarkerSet()) {
- n.setPruningMarker();
- plans.add(n);
- }
- }
- }
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Handling of branches
- // --------------------------------------------------------------------------------------------
-
- public boolean hasUnclosedBranches() {
- return this.openBranches != null && !this.openBranches.isEmpty();
- }
-
- public Set<OptimizerNode> getClosedBranchingNodes() {
- return this.closedBranchingNodes;
- }
-
- public List<UnclosedBranchDescriptor> getOpenBranches() {
- return this.openBranches;
- }
-
-
- protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection toParent) {
- if (this.outgoingConnections.size() == 1) {
- // return our own stack of open branches, because nothing is added
- if (this.openBranches == null || this.openBranches.isEmpty()) {
- return Collections.emptyList();
- } else {
- return new ArrayList<UnclosedBranchDescriptor>(this.openBranches);
- }
- }
- else if (this.outgoingConnections.size() > 1) {
- // we branch add a branch info to the stack
- List<UnclosedBranchDescriptor> branches = new ArrayList<UnclosedBranchDescriptor>(4);
- if (this.openBranches != null) {
- branches.addAll(this.openBranches);
- }
-
- // find out, which output number the connection to the parent
- int num;
- for (num = 0; num < this.outgoingConnections.size(); num++) {
- if (this.outgoingConnections.get(num) == toParent) {
- break;
- }
- }
- if (num >= this.outgoingConnections.size()) {
- throw new CompilerException("Error in compiler: "
- + "Parent to get branch info for is not contained in the outgoing connections.");
- }
-
- // create the description and add it
- long bitvector = 0x1L << num;
- branches.add(new UnclosedBranchDescriptor(this, bitvector));
- return branches;
- }
- else {
- throw new CompilerException(
- "Error in compiler: Cannot get branch info for successor in a node with no successors.");
- }
- }
-
-
- protected void removeClosedBranches(List<UnclosedBranchDescriptor> openList) {
- if (openList == null || openList.isEmpty() || this.closedBranchingNodes == null || this.closedBranchingNodes.isEmpty()) {
- return;
- }
-
- Iterator<UnclosedBranchDescriptor> it = openList.iterator();
- while (it.hasNext()) {
- if (this.closedBranchingNodes.contains(it.next().getBranchingNode())) {
- //this branch was already closed --> remove it from the list
- it.remove();
- }
- }
- }
-
- protected void addClosedBranches(Set<OptimizerNode> alreadyClosed) {
- if (alreadyClosed == null || alreadyClosed.isEmpty()) {
- return;
- }
-
- if (this.closedBranchingNodes == null) {
- this.closedBranchingNodes = new HashSet<OptimizerNode>(alreadyClosed);
- } else {
- this.closedBranchingNodes.addAll(alreadyClosed);
- }
- }
-
- protected void addClosedBranch(OptimizerNode alreadyClosed) {
- if (this.closedBranchingNodes == null) {
- this.closedBranchingNodes = new HashSet<OptimizerNode>();
- }
-
- this.closedBranchingNodes.add(alreadyClosed);
- }
-
- /**
- * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
- * alternative plans are comparable, if
- *
- * a) There is no branch in the sub-plan of this node
- * b) Both candidates have the same candidate as the child at the last open branch.
- *
- * @param plan1 The root node of the first candidate plan.
- * @param plan2 The root node of the second candidate plan.
- * @return True if the nodes are branch compatible in the inputs.
- */
- protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
- if (plan1 == null || plan2 == null) {
- throw new NullPointerException();
- }
-
- // if there is no open branch, the children are always compatible.
- // in most plans, that will be the dominant case
- if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
- return true;
- }
-
- for (OptimizerNode joinedBrancher : hereJoinedBranches) {
- final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
- final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
-
- if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * The node IDs are assigned in graph-traversal order (pre-order), hence, each list is
- * sorted by ID in ascending order and all consecutive lists start with IDs in ascending order.
- *
- * @param markJoinedBranchesAsPipelineBreaking True, if the
- */
- protected final boolean mergeLists(List<UnclosedBranchDescriptor> child1open,
- List<UnclosedBranchDescriptor> child2open,
- List<UnclosedBranchDescriptor> result,
- boolean markJoinedBranchesAsPipelineBreaking) {
-
- //remove branches which have already been closed
- removeClosedBranches(child1open);
- removeClosedBranches(child2open);
-
- result.clear();
-
- // check how many open branches we have. the cases:
- // 1) if both are null or empty, the result is null
- // 2) if one side is null (or empty), the result is the other side.
- // 3) both are set, then we need to merge.
- if (child1open == null || child1open.isEmpty()) {
- if(child2open != null && !child2open.isEmpty()) {
- result.addAll(child2open);
- }
- return false;
- }
-
- if (child2open == null || child2open.isEmpty()) {
- result.addAll(child1open);
- return false;
- }
-
- int index1 = child1open.size() - 1;
- int index2 = child2open.size() - 1;
-
- boolean didCloseABranch = false;
-
- // as both lists (child1open and child2open) are sorted in ascending ID order
- // we can do a merge-join-like loop which preserved the order in the result list
- // and eliminates duplicates
- while (index1 >= 0 || index2 >= 0) {
- int id1 = -1;
- int id2 = index2 >= 0 ? child2open.get(index2).getBranchingNode().getId() : -1;
-
- while (index1 >= 0 && (id1 = child1open.get(index1).getBranchingNode().getId()) > id2) {
- result.add(child1open.get(index1));
- index1--;
- }
- while (index2 >= 0 && (id2 = child2open.get(index2).getBranchingNode().getId()) > id1) {
- result.add(child2open.get(index2));
- index2--;
- }
-
- // match: they share a common branching child
- if (id1 == id2) {
- didCloseABranch = true;
-
- // if this is the latest common child, remember it
- OptimizerNode currBanchingNode = child1open.get(index1).getBranchingNode();
-
- long vector1 = child1open.get(index1).getJoinedPathsVector();
- long vector2 = child2open.get(index2).getJoinedPathsVector();
-
- // check if this is the same descriptor, (meaning that it contains the same paths)
- // if it is the same, add it only once, otherwise process the join of the paths
- if (vector1 == vector2) {
- result.add(child1open.get(index1));
- }
- else {
- // we merge (re-join) a branch
-
- // mark the branch as a point where we break the pipeline
- if (markJoinedBranchesAsPipelineBreaking) {
- currBanchingNode.markAllOutgoingConnectionsAsPipelineBreaking();
- }
-
- if (this.hereJoinedBranches == null) {
- this.hereJoinedBranches = new ArrayList<OptimizerNode>(2);
- }
- this.hereJoinedBranches.add(currBanchingNode);
-
- // see, if this node closes the branch
- long joinedInputs = vector1 | vector2;
-
- // this is 2^size - 1, which is all bits set at positions 0..size-1
- long allInputs = (0x1L << currBanchingNode.getOutgoingConnections().size()) - 1;
-
- if (joinedInputs == allInputs) {
- // closed - we can remove it from the stack
- addClosedBranch(currBanchingNode);
- } else {
- // not quite closed
- result.add(new UnclosedBranchDescriptor(currBanchingNode, joinedInputs));
- }
- }
-
- index1--;
- index2--;
- }
- }
-
- // merged. now we need to reverse the list, because we added the elements in reverse order
- Collections.reverse(result);
- return didCloseABranch;
- }
-
- @Override
- public OptimizerNode getOptimizerNode() {
- return this;
- }
-
- @Override
- public PlanNode getPlanNode() {
- return null;
- }
-
- @Override
- public Iterable<DumpableConnection<OptimizerNode>> getDumpableInputs() {
- List<DumpableConnection<OptimizerNode>> allInputs = new ArrayList<DumpableConnection<OptimizerNode>>();
-
- allInputs.addAll(getIncomingConnections());
- allInputs.addAll(getBroadcastConnections());
-
- return allInputs;
- }
-
- @Override
- public String toString() {
- StringBuilder bld = new StringBuilder();
-
- bld.append(getName());
- bld.append(" (").append(getOperator().getName()).append(") ");
-
- int i = 1;
- for (DagConnection conn : getIncomingConnections()) {
- String shipStrategyName = conn.getShipStrategy() == null ? "null" : conn.getShipStrategy().name();
- bld.append('(').append(i++).append(":").append(shipStrategyName).append(')');
- }
-
- return bld.toString();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Description of an unclosed branch. An unclosed branch is when the data flow branched (one operator's
- * result is consumed by multiple targets), but these different branches (targets) have not been joined
- * together.
- */
- public static final class UnclosedBranchDescriptor {
-
- protected OptimizerNode branchingNode;
-
- protected long joinedPathsVector;
-
- /**
- * Creates a new branching descriptor.
- *
- * @param branchingNode The node where the branch occurred (teh node with multiple outputs).
- * @param joinedPathsVector A bit vector describing which branches are tracked by this descriptor.
- * The bit vector is one, where the branch is tracked, zero otherwise.
- */
- protected UnclosedBranchDescriptor(OptimizerNode branchingNode, long joinedPathsVector) {
- this.branchingNode = branchingNode;
- this.joinedPathsVector = joinedPathsVector;
- }
-
- public OptimizerNode getBranchingNode() {
- return this.branchingNode;
- }
-
- public long getJoinedPathsVector() {
- return this.joinedPathsVector;
- }
-
- @Override
- public String toString() {
- return "(" + this.branchingNode.getOperator() + ") [" + this.joinedPathsVector + "]";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
deleted file mode 100644
index 5c811b0..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- * The optimizer's internal representation of a <i>Partition</i> operator node.
- */
-public class PartitionNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- public PartitionNode(PartitionOperatorBase<?> operator) {
- super(operator);
-
- OperatorDescriptorSingle descr = new PartitionDescriptor(
- this.getOperator().getPartitionMethod(), this.keys, operator.getCustomPartitioner());
- this.possibleProperties = Collections.singletonList(descr);
- }
-
- @Override
- public PartitionOperatorBase<?> getOperator() {
- return (PartitionOperatorBase<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Partition";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // partitioning does not change the number of records
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static class PartitionDescriptor extends OperatorDescriptorSingle {
-
- private final PartitionMethod pMethod;
- private final Partitioner<?> customPartitioner;
-
- public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Partitioner<?> customPartitioner) {
- super(pKeys);
-
- this.pMethod = pMethod;
- this.customPartitioner = customPartitioner;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.UNARY_NO_OP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Partition", in, DriverStrategy.UNARY_NO_OP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties rgps = new RequestedGlobalProperties();
-
- switch (this.pMethod) {
- case HASH:
- rgps.setHashPartitioned(this.keys);
- break;
- case REBALANCE:
- rgps.setForceRebalancing();
- break;
- case CUSTOM:
- rgps.setCustomPartitioned(this.keys, this.customPartitioner);
- break;
- case RANGE:
- throw new UnsupportedOperationException("Not yet supported");
- default:
- throw new IllegalArgumentException("Invalid partition method");
- }
-
- return Collections.singletonList(rgps);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- // partitioning does not require any local property.
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- // the partition node is a no-operation operation, such that all global properties are preserved.
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- // the partition node is a no-operation operation, such that all global properties are preserved.
- return lProps;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
deleted file mode 100644
index bbe4607..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/PlanCacheCleaner.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.util.Visitor;
-
-final class PlanCacheCleaner implements Visitor<OptimizerNode> {
-
- static final PlanCacheCleaner INSTANCE = new PlanCacheCleaner();
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- if (visitable.cachedPlans != null && visitable.isOnDynamicPath()) {
- visitable.cachedPlans = null;
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {}
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
deleted file mode 100644
index 1477038..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/ReduceNode.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.AllReduceProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.optimizer.operators.ReduceProperties;
-
-/**
- * The Optimizer representation of a <i>Reduce</i> operator.
- */
-public class ReduceNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- private ReduceNode preReduceUtilityNode;
-
-
- public ReduceNode(ReduceOperatorBase<?, ?> operator) {
- super(operator);
-
- if (this.keys == null) {
- // case of a key-less reducer. force a parallelism of 1
- setDegreeOfParallelism(1);
- }
-
- OperatorDescriptorSingle props = this.keys == null ?
- new AllReduceProperties() :
- new ReduceProperties(this.keys, operator.getCustomPartitioner());
-
- this.possibleProperties = Collections.singletonList(props);
- }
-
- public ReduceNode(ReduceNode reducerToCopyForCombiner) {
- super(reducerToCopyForCombiner);
-
- this.possibleProperties = Collections.emptyList();
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public ReduceOperatorBase<?, ?> getOperator() {
- return (ReduceOperatorBase<?, ?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Reduce";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- // --------------------------------------------------------------------------------------------
- // Estimates
- // --------------------------------------------------------------------------------------------
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // no real estimates possible for a reducer.
- }
-
- public ReduceNode getCombinerUtilityNode() {
- if (this.preReduceUtilityNode == null) {
- this.preReduceUtilityNode = new ReduceNode(this);
-
- // we conservatively assume the combiner returns the same data size as it consumes
- this.preReduceUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- this.preReduceUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- }
- return this.preReduceUtilityNode;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
deleted file mode 100644
index cc12bb8..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SingleInputNode.java
+++ /dev/null
@@ -1,518 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.InterestingProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.NamedChannel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport;
-import org.apache.flink.optimizer.util.NoOpUnaryUdfOp;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-
-import com.google.common.collect.Sets;
-
-/**
- * A node in the optimizer's program representation for an operation with a single input.
- *
- * This class contains all the generic logic for handling branching flows, as well as to
- * enumerate candidate execution plans. The subclasses for specific operators simply add logic
- * for cost estimates and specify possible strategies for their execution.
- */
-public abstract class SingleInputNode extends OptimizerNode {
-
- protected final FieldSet keys; // The set of key fields
-
- protected DagConnection inConn; // the input of the node
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new node with a single input for the optimizer plan.
- *
- * @param programOperator The PACT that the node represents.
- */
- protected SingleInputNode(SingleInputOperator<?, ?, ?> programOperator) {
- super(programOperator);
-
- int[] k = programOperator.getKeyColumns(0);
- this.keys = k == null || k.length == 0 ? null : new FieldSet(k);
- }
-
- protected SingleInputNode(FieldSet keys) {
- super(NoOpUnaryUdfOp.INSTANCE);
- this.keys = keys;
- }
-
- protected SingleInputNode() {
- super(NoOpUnaryUdfOp.INSTANCE);
- this.keys = null;
- }
-
- protected SingleInputNode(SingleInputNode toCopy) {
- super(toCopy);
-
- this.keys = toCopy.keys;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public SingleInputOperator<?, ?, ?> getOperator() {
- return (SingleInputOperator<?, ?, ?>) super.getOperator();
- }
-
- /**
- * Gets the input of this operator.
- *
- * @return The input.
- */
- public DagConnection getIncomingConnection() {
- return this.inConn;
- }
-
- /**
- * Sets the connection through which this node receives its input.
- *
- * @param inConn The input connection to set.
- */
- public void setIncomingConnection(DagConnection inConn) {
- this.inConn = inConn;
- }
-
- /**
- * Gets the predecessor of this node.
- *
- * @return The predecessor of this node.
- */
- public OptimizerNode getPredecessorNode() {
- if (this.inConn != null) {
- return this.inConn.getSource();
- } else {
- return null;
- }
- }
-
- @Override
- public List<DagConnection> getIncomingConnections() {
- return Collections.singletonList(this.inConn);
- }
-
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return getOperator().getSemanticProperties();
- }
-
-
- @Override
- public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
- throws CompilerException
- {
- // see if an internal hint dictates the strategy to use
- final Configuration conf = getOperator().getParameters();
- final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
- final ShipStrategyType preSet;
-
- if (shipStrategy != null) {
- if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
- preSet = ShipStrategyType.PARTITION_HASH;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
- preSet = ShipStrategyType.PARTITION_RANGE;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
- preSet = ShipStrategyType.FORWARD;
- } else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
- preSet = ShipStrategyType.PARTITION_RANDOM;
- } else {
- throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
- }
- } else {
- preSet = null;
- }
-
- // get the predecessor node
- Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
-
- OptimizerNode pred;
- DagConnection conn;
- if (children == null) {
- throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
- } else {
- pred = contractToNode.get(children);
- conn = new DagConnection(pred, this, defaultExchangeMode);
- if (preSet != null) {
- conn.setShipStrategy(preSet);
- }
- }
-
- // create the connection and add it
- setIncomingConnection(conn);
- pred.addOutgoingConnection(conn);
- }
-
- // --------------------------------------------------------------------------------------------
- // Properties and Optimization
- // --------------------------------------------------------------------------------------------
-
- protected abstract List<OperatorDescriptorSingle> getPossibleProperties();
-
- @Override
- public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
- // get what we inherit and what is preserved by our user code
- final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
-
- // add all properties relevant to this node
- for (OperatorDescriptorSingle dps : getPossibleProperties()) {
- for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
-
- if (gp.getPartitioning().isPartitionedOnKey()) {
- // make sure that among the same partitioning types, we do not push anything down that has fewer key fields
-
- for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
- if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
- props.getGlobalProperties().remove(contained);
- break;
- }
- }
- }
-
- props.addGlobalProperties(gp);
- }
-
- for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
- props.addLocalProperties(lp);
- }
- }
- this.inConn.setInterestingProperties(props);
-
- for (DagConnection conn : getBroadcastConnections()) {
- conn.setInterestingProperties(new InterestingProperties());
- }
- }
-
-
- @Override
- public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
- // check if we have a cached version
- if (this.cachedPlans != null) {
- return this.cachedPlans;
- }
-
- boolean childrenSkippedDueToReplicatedInput = false;
-
- // calculate alternative sub-plans for predecessor
- final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
- final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties();
-
- // calculate alternative sub-plans for broadcast inputs
- final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
- List<DagConnection> broadcastConnections = getBroadcastConnections();
- List<String> broadcastConnectionNames = getBroadcastConnectionNames();
-
- for (int i = 0; i < broadcastConnections.size(); i++ ) {
- DagConnection broadcastConnection = broadcastConnections.get(i);
- String broadcastConnectionName = broadcastConnectionNames.get(i);
- List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
-
- // wrap the plan candidates in named channels
- HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
- for (PlanNode plan: broadcastPlanCandidates) {
- NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
- DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(),
- ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
- c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
- broadcastChannels.add(c);
- }
- broadcastPlanChannels.add(broadcastChannels);
- }
-
- final RequestedGlobalProperties[] allValidGlobals;
- {
- Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>();
- for (OperatorDescriptorSingle ods : getPossibleProperties()) {
- pairs.addAll(ods.getPossibleGlobalProperties());
- }
- allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]);
- }
- final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
-
- final ExecutionMode executionMode = this.inConn.getDataExchangeMode();
-
- final int dop = getParallelism();
- final int inDop = getPredecessorNode().getParallelism();
- final boolean dopChange = inDop != dop;
-
- final boolean breaksPipeline = this.inConn.isBreakingPipeline();
-
- // create all candidates
- for (PlanNode child : subPlans) {
-
- if (child.getGlobalProperties().isFullyReplicated()) {
- // fully replicated input is always locally forwarded if DOP is not changed
- if (dopChange) {
- // can not continue with this child
- childrenSkippedDueToReplicatedInput = true;
- continue;
- } else {
- this.inConn.setShipStrategy(ShipStrategyType.FORWARD);
- }
- }
-
- if (this.inConn.getShipStrategy() == null) {
- // pick the strategy ourselves
- for (RequestedGlobalProperties igps: intGlobal) {
- final Channel c = new Channel(child, this.inConn.getMaterializationMode());
- igps.parameterizeChannel(c, dopChange, executionMode, breaksPipeline);
-
- // if the DOP changed, make sure that we cancel out properties, unless the
- // ship strategy preserves/establishes them even under changing DOPs
- if (dopChange && !c.getShipStrategy().isNetworkStrategy()) {
- c.getGlobalProperties().reset();
- }
-
- // check whether we meet any of the accepted properties
- // we may remove this check, when we do a check to not inherit
- // requested global properties that are incompatible with all possible
- // requested properties
- for (RequestedGlobalProperties rgps: allValidGlobals) {
- if (rgps.isMetBy(c.getGlobalProperties())) {
- c.setRequiredGlobalProps(rgps);
- addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
- break;
- }
- }
- }
- } else {
- // hint fixed the strategy
- final Channel c = new Channel(child, this.inConn.getMaterializationMode());
- final ShipStrategyType shipStrategy = this.inConn.getShipStrategy();
- final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);
-
- if (this.keys != null) {
- c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode);
- } else {
- c.setShipStrategy(shipStrategy, exMode);
- }
-
- if (dopChange) {
- c.adjustGlobalPropertiesForFullParallelismChange();
- }
-
- // check whether we meet any of the accepted properties
- for (RequestedGlobalProperties rgps: allValidGlobals) {
- if (rgps.isMetBy(c.getGlobalProperties())) {
- addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
- break;
- }
- }
- }
- }
-
- if(outputPlans.isEmpty()) {
- if(childrenSkippedDueToReplicatedInput) {
- throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input.");
- } else {
- throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
- }
- }
-
- // cost and prune the plans
- for (PlanNode node : outputPlans) {
- estimator.costOperator(node);
- }
- prunePlanAlternatives(outputPlans);
- outputPlans.trimToSize();
-
- this.cachedPlans = outputPlans;
- return outputPlans;
- }
-
- protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
- List<PlanNode> target, CostEstimator estimator)
- {
- for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
- final Channel in = template.clone();
- ilp.parameterizeChannel(in);
-
- // instantiate a candidate, if the instantiated local properties meet one possible local property set
- outer:
- for (OperatorDescriptorSingle dps: getPossibleProperties()) {
- for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
- if (ilps.isMetBy(in.getLocalProperties())) {
- in.setRequiredLocalProps(ilps);
- instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
- break outer;
- }
- }
- }
- }
- }
-
- protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
- List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
- {
- final PlanNode inputSource = in.getSource();
-
- for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
-
- boolean validCombination = true;
- boolean requiresPipelinebreaker = false;
-
- // check whether the broadcast inputs use the same plan candidate at the branching point
- for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
- NamedChannel nc = broadcastChannelsCombination.get(i);
- PlanNode bcSource = nc.getSource();
-
- // check branch compatibility against input
- if (!areBranchCompatible(bcSource, inputSource)) {
- validCombination = false;
- break;
- }
-
- // check branch compatibility against all other broadcast variables
- for (int k = 0; k < i; k++) {
- PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
-
- if (!areBranchCompatible(bcSource, otherBcSource)) {
- validCombination = false;
- break;
- }
- }
-
- // check if there is a common predecessor and whether there is a dam on the way to all common predecessors
- if (this.hereJoinedBranches != null) {
- for (OptimizerNode brancher : this.hereJoinedBranches) {
- PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
-
- if (candAtBrancher == null) {
- // closed branch between two broadcast variables
- continue;
- }
-
- SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
- if (res == NOT_FOUND) {
- throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
- } else if (res == FOUND_SOURCE) {
- requiresPipelinebreaker = true;
- break;
- } else if (res == FOUND_SOURCE_AND_DAM) {
- // good
- } else {
- throw new CompilerException();
- }
- }
- }
- }
-
- if (!validCombination) {
- continue;
- }
-
- if (requiresPipelinebreaker) {
- in.setTempMode(in.getTempMode().makePipelineBreaker());
- }
-
- final SingleInputPlanNode node = dps.instantiate(in, this);
- node.setBroadcastInputs(broadcastChannelsCombination);
-
- // compute how the strategy affects the properties
- GlobalProperties gProps = in.getGlobalProperties().clone();
- LocalProperties lProps = in.getLocalProperties().clone();
- gProps = dps.computeGlobalProperties(gProps);
- lProps = dps.computeLocalProperties(lProps);
-
- SemanticProperties props = this.getSemanticProperties();
- // filter by the user code field copies
- gProps = gProps.filterBySemanticProperties(props, 0);
- lProps = lProps.filterBySemanticProperties(props, 0);
-
- // apply
- node.initProperties(gProps, lProps);
- node.updatePropertiesWithUniqueSets(getUniqueFields());
- target.add(node);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Branch Handling
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- addClosedBranches(getPredecessorNode().closedBranchingNodes);
- List<UnclosedBranchDescriptor> fromInput = getPredecessorNode().getBranchesForParent(this.inConn);
-
- // handle the data flow branching for the broadcast inputs
- List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(fromInput);
-
- this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
- }
-
- // --------------------------------------------------------------------------------------------
- // Miscellaneous
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void accept(Visitor<OptimizerNode> visitor) {
- if (visitor.preVisit(this)) {
- if (getPredecessorNode() != null) {
- getPredecessorNode().accept(visitor);
- } else {
- throw new CompilerException();
- }
- for (DagConnection connection : getBroadcastConnections()) {
- connection.getSource().accept(visitor);
- }
- visitor.postVisit(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
deleted file mode 100644
index 40725ba..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SinkJoiner.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
-import org.apache.flink.optimizer.operators.UtilSinkJoinOpDescriptor;
-import org.apache.flink.optimizer.util.NoOpBinaryUdfOp;
-import org.apache.flink.types.Nothing;
-
-/**
- * This class represents a utility node that is not part of the actual plan.
- * It is used for plans with multiple data sinks to transform it into a plan with
- * a single root node. That way, the code that makes sure no costs are double-counted and that
- * candidate selection works correctly with nodes that have multiple outputs is transparently reused.
- */
-public class SinkJoiner extends TwoInputNode {
-
- public SinkJoiner(OptimizerNode input1, OptimizerNode input2) {
- super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));
-
- DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED);
- DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED);
-
- this.input1 = conn1;
- this.input2 = conn2;
-
- setDegreeOfParallelism(1);
- }
-
- @Override
- public String getName() {
- return "Internal Utility Node";
- }
-
- @Override
- public List<DagConnection> getOutgoingConnections() {
- return Collections.emptyList();
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
- addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
-
- List<UnclosedBranchDescriptor> pred1branches = getFirstPredecessorNode().openBranches;
- List<UnclosedBranchDescriptor> pred2branches = getSecondPredecessorNode().openBranches;
-
- // if the predecessors do not have branches, then we have multiple sinks that do not originate from
- // a common data flow.
- if (pred1branches == null || pred1branches.isEmpty()) {
-
- this.openBranches = (pred2branches == null || pred2branches.isEmpty()) ?
- Collections.<UnclosedBranchDescriptor>emptyList() : // both empty - disconnected flow
- pred2branches;
- }
- else if (pred2branches == null || pred2branches.isEmpty()) {
- this.openBranches = pred1branches;
- }
- else {
- // copy the lists and merge
- List<UnclosedBranchDescriptor> result1 = new ArrayList<UnclosedBranchDescriptor>(pred1branches);
- List<UnclosedBranchDescriptor> result2 = new ArrayList<UnclosedBranchDescriptor>(pred2branches);
-
- ArrayList<UnclosedBranchDescriptor> result = new ArrayList<UnclosedBranchDescriptor>();
- mergeLists(result1, result2, result, false);
-
- this.openBranches = result.isEmpty() ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
- }
- }
-
- @Override
- protected List<OperatorDescriptorDual> getPossibleProperties() {
- return Collections.<OperatorDescriptorDual>singletonList(new UtilSinkJoinOpDescriptor());
- }
-
- @Override
- public void computeOutputEstimates(DataStatistics statistics) {
- // nothing to be done here
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // no estimates needed at this point
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
deleted file mode 100644
index 1292cf5..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SolutionSetNode.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-
-/**
- * The optimizer's internal representation of the solution set of a workset iteration.
- */
-public class SolutionSetNode extends AbstractPartialSolutionNode {
-
- private final WorksetIterationNode iterationNode;
-
-
- public SolutionSetNode(SolutionSetPlaceHolder<?> psph, WorksetIterationNode iterationNode) {
- super(psph);
- this.iterationNode = iterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
- this.cachedPlans = Collections.<PlanNode>singletonList(new SolutionSetPlanNode(this, "SolutionSet("+this.getOperator().getName()+")", gProps, lProps, initialInput));
- }
-
- public SolutionSetPlanNode getCurrentSolutionSetPlanNode() {
- if (this.cachedPlans != null) {
- return (SolutionSetPlanNode) this.cachedPlans.get(0);
- } else {
- throw new IllegalStateException();
- }
- }
-
- public WorksetIterationNode getIterationNode() {
- return this.iterationNode;
- }
-
- @Override
- public void computeOutputEstimates(DataStatistics statistics) {
- copyEstimates(this.iterationNode.getInitialSolutionSetPredecessorNode());
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the contract object for this data source node.
- *
- * @return The contract.
- */
- @Override
- public SolutionSetPlaceHolder<?> getOperator() {
- return (SolutionSetPlaceHolder<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Solution Set";
- }
-
- @Override
- public void computeUnclosedBranchStack() {
- if (this.openBranches != null) {
- return;
- }
-
- DagConnection solutionSetInput = this.iterationNode.getFirstIncomingConnection();
- OptimizerNode solutionSetSource = solutionSetInput.getSource();
-
- addClosedBranches(solutionSetSource.closedBranchingNodes);
- List<UnclosedBranchDescriptor> fromInput = solutionSetSource.getBranchesForParent(solutionSetInput);
- this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
deleted file mode 100644
index 83bc39a..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/SortPartitionNode.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dag;
-
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
-import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The optimizer's internal representation of a <i>SortPartition</i> operator node.
- */
-public class SortPartitionNode extends SingleInputNode {
-
- private final List<OperatorDescriptorSingle> possibleProperties;
-
- public SortPartitionNode(SortPartitionOperatorBase<?> operator) {
- super(operator);
-
- OperatorDescriptorSingle descr = new SortPartitionDescriptor(operator.getPartitionOrdering());
- this.possibleProperties = Collections.singletonList(descr);
- }
-
- @Override
- public SortPartitionOperatorBase<?> getOperator() {
- return (SortPartitionOperatorBase<?>) super.getOperator();
- }
-
- @Override
- public String getName() {
- return "Sort-Partition";
- }
-
- @Override
- protected List<OperatorDescriptorSingle> getPossibleProperties() {
- return this.possibleProperties;
- }
-
- @Override
- protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
- // sorting does not change the number of records
- this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
- this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
- }
-
- @Override
- public SemanticProperties getSemanticProperties() {
- return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static class SortPartitionDescriptor extends OperatorDescriptorSingle {
-
- private Ordering partitionOrder;
-
- public SortPartitionDescriptor(Ordering partitionOrder) {
- this.partitionOrder = partitionOrder;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.UNARY_NO_OP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Sort-Partition", in, DriverStrategy.UNARY_NO_OP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- // sort partition does not require any global property
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- // set partition order as required local property
- RequestedLocalProperties rlp = new RequestedLocalProperties();
- rlp.setOrdering(this.partitionOrder);
-
- return Collections.singletonList(rlp);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- // sort partition is a no-operation operation, such that all global properties are preserved.
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- // sort partition is a no-operation operation, such that all global properties are preserved.
- return lProps;
- }
- }
-}
[02/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
new file mode 100644
index 0000000..9f2c467
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalPropertiesFilteringTest.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.operators.DualInputSemanticProperties;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.junit.Test;
+
+
+public class RequestedGlobalPropertiesFilteringTest {
+
+ private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
+ new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ @Test(expected = NullPointerException.class)
+ public void testNullProps() {
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0,1,2));
+
+ rgProps.filterBySemanticProperties(null, 0);
+ }
+
+ @Test
+ public void testEraseAll1() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0,1,2));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testEraseAll2() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"3;4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0, 1, 2));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testHashPartitioningPreserved1() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;3;4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
+ assertNotNull(filtered.getPartitionedFields());
+ assertEquals(3, filtered.getPartitionedFields().size());
+ assertTrue(filtered.getPartitionedFields().contains(0));
+ assertTrue(filtered.getPartitionedFields().contains(3));
+ assertTrue(filtered.getPartitionedFields().contains(4));
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testHashPartitioningPreserved2() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
+ assertNotNull(filtered.getPartitionedFields());
+ assertEquals(3, filtered.getPartitionedFields().size());
+ assertTrue(filtered.getPartitionedFields().contains(1));
+ assertTrue(filtered.getPartitionedFields().contains(2));
+ assertTrue(filtered.getPartitionedFields().contains(7));
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testHashPartitioningErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setHashPartitioned(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testAnyPartitioningPreserved1() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;3;4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
+ assertNotNull(filtered.getPartitionedFields());
+ assertEquals(3, filtered.getPartitionedFields().size());
+ assertTrue(filtered.getPartitionedFields().contains(0));
+ assertTrue(filtered.getPartitionedFields().contains(3));
+ assertTrue(filtered.getPartitionedFields().contains(4));
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testAnyPartitioningPreserved2() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
+ assertNotNull(filtered.getPartitionedFields());
+ assertEquals(3, filtered.getPartitionedFields().size());
+ assertTrue(filtered.getPartitionedFields().contains(1));
+ assertTrue(filtered.getPartitionedFields().contains(2));
+ assertTrue(filtered.getPartitionedFields().contains(7));
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testAnyPartitioningErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testRangePartitioningPreserved1() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;3;6"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(3, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setRangePartitioned(o);
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
+ assertNotNull(filtered.getOrdering());
+ assertEquals(3, filtered.getOrdering().getNumberOfFields());
+ assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
+ assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
+ assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
+ assertEquals(LongValue.class, filtered.getOrdering().getType(0));
+ assertEquals(IntValue.class, filtered.getOrdering().getType(1));
+ assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
+ assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
+ assertNull(filtered.getPartitionedFields());
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ }
+
+ @Test
+ public void testRangePartitioningPreserved2() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(3, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setRangePartitioned(o);
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
+ assertNotNull(filtered.getOrdering());
+ assertEquals(3, filtered.getOrdering().getNumberOfFields());
+ assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
+ assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
+ assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
+ assertEquals(LongValue.class, filtered.getOrdering().getType(0));
+ assertEquals(IntValue.class, filtered.getOrdering().getType(1));
+ assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
+ assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
+ assertNull(filtered.getPartitionedFields());
+ assertNull(filtered.getDataDistribution());
+ assertNull(filtered.getCustomPartitioner());
+ }
+
+ @Test
+ public void testRangePartitioningPreserved3() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);
+
+ DataDistribution dd = new MockDistribution();
+ Ordering o = new Ordering();
+ o.appendOrdering(3, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setRangePartitioned(o, dd);
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNotNull(filtered);
+ assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
+ assertNotNull(filtered.getOrdering());
+ assertEquals(3, filtered.getOrdering().getNumberOfFields());
+ assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
+ assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
+ assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
+ assertEquals(LongValue.class, filtered.getOrdering().getType(0));
+ assertEquals(IntValue.class, filtered.getOrdering().getType(1));
+ assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
+ assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
+ assertNotNull(filtered.getDataDistribution());
+ assertEquals(dd, filtered.getDataDistribution());
+ assertNull(filtered.getPartitionedFields());
+ assertNull(filtered.getCustomPartitioner());
+ }
+
+ @Test
+ public void testRangePartitioningErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(3, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setRangePartitioned(o);
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testCustomPartitioningErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setCustomPartitioned(new FieldSet(0, 1, 2), new MockPartitioner());
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testRandomDistributionErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setRandomPartitioning();
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testReplicationErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setFullyReplicated();
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testRebalancingErased() {
+
+ SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"0;1;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
+ rgProps.setForceRebalancing();
+
+ RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testDualHashPartitioningPreserved() {
+
+ DualInputSemanticProperties dprops = new DualInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"0;2;4"}, new String[]{"1->3;4->6;3->7"},
+ null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties gprops1 = new RequestedGlobalProperties();
+ RequestedGlobalProperties gprops2 = new RequestedGlobalProperties();
+ gprops1.setHashPartitioned(new FieldSet(2, 0, 4));
+ gprops2.setHashPartitioned(new FieldSet(3, 6, 7));
+ RequestedGlobalProperties filtered1 = gprops1.filterBySemanticProperties(dprops, 0);
+ RequestedGlobalProperties filtered2 = gprops2.filterBySemanticProperties(dprops, 1);
+
+ assertNotNull(filtered1);
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered1.getPartitioning());
+ assertNotNull(filtered1.getPartitionedFields());
+ assertEquals(3, filtered1.getPartitionedFields().size());
+ assertTrue(filtered1.getPartitionedFields().contains(0));
+ assertTrue(filtered1.getPartitionedFields().contains(2));
+ assertTrue(filtered1.getPartitionedFields().contains(4));
+ assertNull(filtered1.getOrdering());
+ assertNull(filtered1.getCustomPartitioner());
+ assertNull(filtered1.getDataDistribution());
+
+ assertNotNull(filtered2);
+ assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered2.getPartitioning());
+ assertNotNull(filtered2.getPartitionedFields());
+ assertEquals(3, filtered2.getPartitionedFields().size());
+ assertTrue(filtered2.getPartitionedFields().contains(1));
+ assertTrue(filtered2.getPartitionedFields().contains(3));
+ assertTrue(filtered2.getPartitionedFields().contains(4));
+ assertNull(filtered2.getOrdering());
+ assertNull(filtered2.getCustomPartitioner());
+ assertNull(filtered2.getDataDistribution());
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testInvalidInputIndex() {
+ SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedGlobalProperties gprops = new RequestedGlobalProperties();
+ gprops.setHashPartitioned(new FieldSet(0,1));
+
+ gprops.filterBySemanticProperties(sprops, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
new file mode 100644
index 0000000..0cede0e
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataproperties/RequestedLocalPropertiesFilteringTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.operators.DualInputSemanticProperties;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.types.ByteValue;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.junit.Test;
+
+public class RequestedLocalPropertiesFilteringTest {
+
+ private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
+ new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ @Test(expected = NullPointerException.class)
+ public void testNullProps() {
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(0, 2, 3));
+
+ rlProp.filterBySemanticProperties(null, 0);
+ }
+
+ @Test
+ public void testAllErased() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(0, 2, 3));
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testGroupingPreserved1() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(0, 2, 3));
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNotNull(filtered);
+ assertNotNull(filtered.getGroupedFields());
+ assertEquals(3, filtered.getGroupedFields().size());
+ assertTrue(filtered.getGroupedFields().contains(0));
+ assertTrue(filtered.getGroupedFields().contains(2));
+ assertTrue(filtered.getGroupedFields().contains(3));
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testGroupingPreserved2() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"3->0;5->2;1->3"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(0, 2, 3));
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNotNull(filtered);
+ assertNotNull(filtered.getGroupedFields());
+ assertEquals(3, filtered.getGroupedFields().size());
+ assertTrue(filtered.getGroupedFields().contains(3));
+ assertTrue(filtered.getGroupedFields().contains(5));
+ assertTrue(filtered.getGroupedFields().contains(1));
+ assertNull(filtered.getOrdering());
+ }
+
+ @Test
+ public void testGroupingErased() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(0, 2, 3));
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testOrderPreserved1() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1;4;6"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(4, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setOrdering(o);
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNotNull(filtered);
+ assertNotNull(filtered.getOrdering());
+ assertEquals(3, filtered.getOrdering().getNumberOfFields());
+ assertEquals(4, filtered.getOrdering().getFieldNumber(0).intValue());
+ assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
+ assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
+ assertEquals(LongValue.class, filtered.getOrdering().getType(0));
+ assertEquals(IntValue.class, filtered.getOrdering().getType(1));
+ assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
+ assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
+ assertNull(filtered.getGroupedFields());
+ }
+
+ @Test
+ public void testOrderPreserved2() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"5->1;0->4;2->6"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(4, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setOrdering(o);
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNotNull(filtered);
+ assertNotNull(filtered.getOrdering());
+ assertEquals(3, filtered.getOrdering().getNumberOfFields());
+ assertEquals(0, filtered.getOrdering().getFieldNumber(0).intValue());
+ assertEquals(5, filtered.getOrdering().getFieldNumber(1).intValue());
+ assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
+ assertEquals(LongValue.class, filtered.getOrdering().getType(0));
+ assertEquals(IntValue.class, filtered.getOrdering().getType(1));
+ assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
+ assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
+ assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
+ assertNull(filtered.getGroupedFields());
+ }
+
+ @Test
+ public void testOrderErased() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);
+
+ Ordering o = new Ordering();
+ o.appendOrdering(4, LongValue.class, Order.DESCENDING);
+ o.appendOrdering(1, IntValue.class, Order.ASCENDING);
+ o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setOrdering(o);
+
+ RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
+
+ assertNull(filtered);
+ }
+
+ @Test
+ public void testDualGroupingPreserved() {
+
+ DualInputSemanticProperties dprops = new DualInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"},
+ null, null, null, null, tupleInfo, tupleInfo, tupleInfo);
+
+ RequestedLocalProperties lprops1 = new RequestedLocalProperties();
+ lprops1.setGroupedFields(new FieldSet(0,3,4));
+
+ RequestedLocalProperties lprops2 = new RequestedLocalProperties();
+ lprops2.setGroupedFields(new FieldSet(7, 1));
+
+ RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
+ RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);
+
+ assertNotNull(filtered1);
+ assertNotNull(filtered1.getGroupedFields());
+ assertEquals(3, filtered1.getGroupedFields().size());
+ assertTrue(filtered1.getGroupedFields().contains(1));
+ assertTrue(filtered1.getGroupedFields().contains(2));
+ assertTrue(filtered1.getGroupedFields().contains(3));
+ assertNull(filtered1.getOrdering());
+
+ assertNotNull(filtered2);
+ assertNotNull(filtered2.getGroupedFields());
+ assertEquals(2, filtered2.getGroupedFields().size());
+ assertTrue(filtered2.getGroupedFields().contains(0));
+ assertTrue(filtered2.getGroupedFields().contains(1));
+ assertNull(filtered2.getOrdering());
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testInvalidInputIndex() {
+
+ SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
+ SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);
+
+ RequestedLocalProperties rlProp = new RequestedLocalProperties();
+ rlProp.setGroupedFields(new FieldSet(1, 4));
+
+ rlProp.filterBySemanticProperties(sProps, 1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
new file mode 100644
index 0000000..b359e6b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DeltaIterationDependenciesTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.CompilerTestBase;
+
+
+@SuppressWarnings({"serial", "unchecked"})
+public class DeltaIterationDependenciesTest extends CompilerTestBase {
+
+ @Test
+ public void testExceptionWhenNewWorksetNotDependentOnWorkset() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> deltaIteration = input.iterateDelta(input, 10,0);
+
+ DataSet<Tuple2<Long, Long>> delta = deltaIteration.getSolutionSet().join(deltaIteration.getWorkset())
+ .where(0).equalTo(0)
+ .projectFirst(1).projectSecond(1);
+
+ DataSet<Tuple2<Long, Long>> nextWorkset = deltaIteration.getSolutionSet().join(input)
+ .where(0).equalTo(0)
+ .projectFirst(1).projectSecond(1);
+
+
+ DataSet<Tuple2<Long, Long>> result = deltaIteration.closeWith(delta, nextWorkset);
+
+ result.print();
+
+ Plan p = env.createProgramPlan();
+ try {
+ compileNoStats(p);
+ fail("Should not be able to compile, since the next workset does not depend on the workset");
+ }
+ catch (CompilerException e) {
+ // good
+ }
+ catch (Exception e) {
+ fail("wrong exception type");
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
new file mode 100644
index 0000000..de02836
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/DistinctAndGroupingOptimizerTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+@SuppressWarnings("serial")
+public class DistinctAndGroupingOptimizerTest extends CompilerTestBase {
+
+ @Test
+ public void testDistinctPreservesPartitioningOfDistinctFields() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(4);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L), new Tuple2<Long, Long>(1L, 1L))
+ .map(new IdentityMapper<Tuple2<Long,Long>>()).setParallelism(4);
+
+ data.distinct(0)
+ .groupBy(0)
+ .sum(1)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode distinctReducer = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+
+ // reducer can be forward, reuses partitioning from distinct
+ assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
+
+ // distinct reducer is partitioned
+ assertEquals(ShipStrategyType.PARTITION_HASH, distinctReducer.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDistinctDestroysPartitioningOfNonDistinctFields() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(4);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L), new Tuple2<Long, Long>(1L, 1L))
+ .map(new IdentityMapper<Tuple2<Long,Long>>()).setParallelism(4);
+
+ data.distinct(1)
+ .groupBy(0)
+ .sum(1)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
+ SingleInputPlanNode distinctReducer = (SingleInputPlanNode) combiner.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
+
+ // reducer must repartition, because it works on a different field
+ assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
+
+ assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
+
+ // distinct reducer is partitioned
+ assertEquals(ShipStrategyType.PARTITION_HASH, distinctReducer.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
new file mode 100644
index 0000000..a683968
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/GroupReduceCompilationTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.GroupReduceOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+
+import static org.junit.Assert.*;
+
+@SuppressWarnings("serial")
+public class GroupReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {
+
+ @Test
+ public void testAllGroupReduceNoCombiner() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
+
+ data.reduceGroup(new RichGroupReduceFunction<Double, Double>() {
+ public void reduce(Iterable<Double> values, Collector<Double> out) {}
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+
+ // the all-reduce has no combiner, when the DOP of the input is one
+
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // check wiring
+ assertEquals(sourceNode, reduceNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that reduce has the right strategy
+ assertEquals(DriverStrategy.ALL_GROUP_REDUCE, reduceNode.getDriverStrategy());
+
+ // check DOP
+ assertEquals(1, sourceNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testAllReduceWithCombiner() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
+
+ GroupReduceOperator<Long, Long> reduced = data.reduceGroup(new RichGroupReduceFunction<Long, Long>() {
+ public void reduce(Iterable<Long> values, Collector<Long> out) {}
+ }).name("reducer");
+
+ reduced.setCombinable(true);
+ reduced.print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.ALL_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.ALL_GROUP_REDUCE_COMBINE, combineNode.getDriverStrategy());
+
+ // check DOP
+ assertEquals(8, sourceNode.getParallelism());
+ assertEquals(8, combineNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void testGroupedReduceWithFieldPositionKeyNonCombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .groupBy(1)
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // check wiring
+ assertEquals(sourceNode, reduceNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(1), reduceNode.getKeys(0));
+ assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGroupedReduceWithFieldPositionKeyCombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
+ .groupBy(1)
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ }).name("reducer");
+
+ reduced.setCombinable(true);
+ reduced.print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(1), reduceNode.getKeys(0));
+ assertEquals(new FieldList(1), combineNode.getKeys(0));
+ assertEquals(new FieldList(1), combineNode.getKeys(1));
+ assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGroupedReduceWithSelectorFunctionKeyNoncombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
+ public String getKey(Tuple2<String, Double> value) { return value.f0; }
+ })
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the key extractors and projectors
+ SingleInputPlanNode keyExtractor = (SingleInputPlanNode) reduceNode.getInput().getSource();
+ SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, keyExtractor.getInput().getSource());
+ assertEquals(keyProjector, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(0), reduceNode.getKeys(0));
+ assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGroupedReduceWithSelectorFunctionKeyCombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
+ .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
+ public String getKey(Tuple2<String, Double> value) { return value.f0; }
+ })
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ }).name("reducer");
+
+ reduced.setCombinable(true);
+ reduced.print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // get the key extractors and projectors
+ SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
+ SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, keyExtractor.getInput().getSource());
+ assertEquals(keyProjector, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(0), reduceNode.getKeys(0));
+ assertEquals(new FieldList(0), combineNode.getKeys(0));
+ assertEquals(new FieldList(0), combineNode.getKeys(1));
+ assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
new file mode 100644
index 0000000..37a8e81
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/IterationCompilerTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class IterationCompilerTest extends CompilerTestBase {
+
+ @Test
+ public void testIdentityIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(43);
+
+ IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
+ iteration.closeWith(iteration).print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEmptyWorksetIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(43);
+
+ DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value){ return null; }
+ });
+
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
+ iter.closeWith(iter.getWorkset(), iter.getWorkset())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testIterationWithUnionRoot() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(43);
+
+ IterativeDataSet<Long> iteration = env.generateSequence(-4, 1000).iterate(100);
+
+ iteration.closeWith(
+ iteration.map(new IdentityMapper<Long>()).union(iteration.map(new IdentityMapper<Long>())))
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ BulkIterationPlanNode iterNode = (BulkIterationPlanNode) sink.getInput().getSource();
+
+ // make sure that the root is part of the dynamic path
+
+ // the "NoOp" that comes after the union.
+ SingleInputPlanNode noop = (SingleInputPlanNode) iterNode.getRootOfStepFunction();
+ NAryUnionPlanNode union = (NAryUnionPlanNode) noop.getInput().getSource();
+
+ assertTrue(noop.isOnDynamicPath());
+ assertTrue(noop.getCostWeight() >= 1);
+
+ assertTrue(union.isOnDynamicPath());
+ assertTrue(union.getCostWeight() >= 1);
+
+ // see that the jobgraph generator can translate this
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testWorksetIterationWithUnionRoot() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(43);
+
+ DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value){ return null; }
+ });
+
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
+ iter.closeWith(
+ iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
+ .union(
+ iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
+ , iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
+ .union(
+ iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
+ )
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
+
+ // make sure that the root is part of the dynamic path
+
+ // the "NoOp"a that come after the union.
+ SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
+ SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
+
+ NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
+ NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
+
+ assertTrue(nextWorksetNoop.isOnDynamicPath());
+ assertTrue(nextWorksetNoop.getCostWeight() >= 1);
+
+ assertTrue(solutionDeltaNoop.isOnDynamicPath());
+ assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
+
+ assertTrue(nextWorksetUnion.isOnDynamicPath());
+ assertTrue(nextWorksetUnion.getCostWeight() >= 1);
+
+ assertTrue(solutionDeltaUnion.isOnDynamicPath());
+ assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
+
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
new file mode 100644
index 0000000..0a62132
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/JoinTranslationTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class JoinTranslationTest extends CompilerTestBase {
+
+ @Test
+ public void testBroadcastHashFirstTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST);
+ assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy());
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBroadcastHashSecondTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_SECOND);
+ assertEquals(ShipStrategyType.FORWARD, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.BROADCAST, node.getInput2().getShipStrategy());
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionHashFirstTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_HASH_FIRST);
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionHashSecondTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_HASH_SECOND);
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitionSortMergeTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.REPARTITION_SORT_MERGE);
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
+ assertEquals(DriverStrategy.MERGE, node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testOptimizerChoosesTest() {
+ try {
+ DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES);
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy());
+ assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() ||
+ DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + ": " + e.getMessage());
+ }
+ }
+
+
+ private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> i1 = env.generateSequence(1, 1000);
+ DataSet<Long> i2 = env.generateSequence(1, 1000);
+
+ i1.join(i2, hint).where(new IdentityKeySelector<Long>()).equalTo(new IdentityKeySelector<Long>()).print();
+
+ Plan plan = env.createProgramPlan();
+
+ // set statistics to the sources
+ plan.accept(new Visitor<Operator<?>>() {
+ @Override
+ public boolean preVisit(Operator<?> visitable) {
+ if (visitable instanceof GenericDataSourceBase) {
+ GenericDataSourceBase<?, ?> source = (GenericDataSourceBase<?, ?>) visitable;
+ setSourceStatistics(source, 10000000, 1000);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void postVisit(Operator<?> visitable) {}
+ });
+
+ OptimizedPlan op = compileWithStats(plan);
+
+ return (DualInputPlanNode) ((SinkPlanNode) op.getDataSinks().iterator().next()).getInput().getSource();
+ }
+
+
+
+ private static final class IdentityKeySelector<T> implements KeySelector<T, T> {
+
+ @Override
+ public T getKey(T value) {
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
new file mode 100644
index 0000000..cd63b72
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/OpenIterationTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class OpenIterationTest extends CompilerTestBase {
+
+ @Test
+ public void testSinkInOpenBulkIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> input = env.generateSequence(1, 10);
+
+ IterativeDataSet<Long> iteration = input.iterate(10);
+
+ DataSet<Long> mapped = iteration.map(new IdentityMapper<Long>());
+
+ mapped.print();
+
+ try {
+ env.createProgramPlan();
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSinkInClosedBulkIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> input = env.generateSequence(1, 10);
+
+ IterativeDataSet<Long> iteration = input.iterate(10);
+
+ DataSet<Long> mapped = iteration.map(new IdentityMapper<Long>());
+
+ iteration.closeWith(mapped).print();
+
+ mapped.print();
+
+ Plan p = env.createProgramPlan();
+
+ try {
+ compileNoStats(p);
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSinkOnSolutionSetDeltaIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
+
+ DataSet<Tuple2<Long, Long>> mapped = iteration.getSolutionSet().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ mapped.print();
+
+ try {
+ env.createProgramPlan();
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testSinkOnWorksetDeltaIteration() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
+
+ DataSet<Tuple2<Long, Long>> mapped = iteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ mapped.print();
+
+ try {
+ env.createProgramPlan();
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testOperationOnSolutionSet() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L,0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 10, 0);
+
+ DataSet<Tuple2<Long, Long>> mapped = iteration.getSolutionSet().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined = iteration.getWorkset().join(mapped)
+ .where(0).equalTo(0).projectFirst(1).projectSecond(0);
+
+ iteration.closeWith(joined, joined)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ try {
+ compileNoStats(p);
+ fail("should throw an exception");
+ }
+ catch (InvalidProgramException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
new file mode 100644
index 0000000..8bb9a76
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/PartitionOperatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import static org.junit.Assert.*;
+
+import java.util.Collections;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class PartitionOperatorTest extends CompilerTestBase {
+
+ @Test
+ public void testPartitionOperatorPreservesFields() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<Long, Long>(0L, 0L)));
+
+ data.partitionCustom(new Partitioner<Long>() {
+ public int partition(Long key, int numPartitions) { return key.intValue(); }
+ }, 1)
+ .groupBy(1)
+ .reduceGroup(new IdentityGroupReducer<Tuple2<Long,Long>>())
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode partitioner = (SingleInputPlanNode) reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
new file mode 100644
index 0000000..0724a9f
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/java/ReduceCompilationTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.java;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.CompilerTestBase;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+import static org.junit.Assert.*;
+
+@SuppressWarnings("serial")
+public class ReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {
+
+ @Test
+ public void testAllReduceNoCombiner() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
+
+ data.reduce(new RichReduceFunction<Double>() {
+
+ @Override
+ public Double reduce(Double value1, Double value2){
+ return value1 + value2;
+ }
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+
+ // the all-reduce has no combiner, when the DOP of the input is one
+
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // check wiring
+ assertEquals(sourceNode, reduceNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check DOP
+ assertEquals(1, sourceNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testAllReduceWithCombiner() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
+
+ data.reduce(new RichReduceFunction<Long>() {
+
+ @Override
+ public Long reduce(Long value1, Long value2){
+ return value1 + value2;
+ }
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.ALL_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.ALL_REDUCE, combineNode.getDriverStrategy());
+
+ // check DOP
+ assertEquals(8, sourceNode.getParallelism());
+ assertEquals(8, combineNode.getParallelism());
+ assertEquals(1, reduceNode.getParallelism());
+ assertEquals(1, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGroupedReduceWithFieldPositionKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .groupBy(1)
+ .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
+ @Override
+ public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
+ return null;
+ }
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, combineNode.getInput().getSource());
+ assertEquals(reduceNode, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(1), reduceNode.getKeys(0));
+ assertEquals(new FieldList(1), combineNode.getKeys(0));
+ assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGroupedReduceWithSelectorFunctionKey() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(8);
+
+ DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
+ .name("source").setParallelism(6);
+
+ data
+ .groupBy(new KeySelector<Tuple2<String,Double>, String>() {
+ public String getKey(Tuple2<String, Double> value) { return value.f0; }
+ })
+ .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
+ @Override
+ public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
+ return null;
+ }
+ }).name("reducer")
+ .print().name("sink");
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
+
+ // get the original nodes
+ SourcePlanNode sourceNode = resolver.getNode("source");
+ SingleInputPlanNode reduceNode = resolver.getNode("reducer");
+ SinkPlanNode sinkNode = resolver.getNode("sink");
+
+ // get the combiner
+ SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
+
+ // get the key extractors and projectors
+ SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
+ SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
+
+ // check wiring
+ assertEquals(sourceNode, keyExtractor.getInput().getSource());
+ assertEquals(keyProjector, sinkNode.getInput().getSource());
+
+ // check that both reduce and combiner have the same strategy
+ assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy());
+ assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy());
+
+ // check the keys
+ assertEquals(new FieldList(0), reduceNode.getKeys(0));
+ assertEquals(new FieldList(0), combineNode.getKeys(0));
+ assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
+
+ // check DOP
+ assertEquals(6, sourceNode.getParallelism());
+ assertEquals(6, keyExtractor.getParallelism());
+ assertEquals(6, combineNode.getParallelism());
+
+ assertEquals(8, reduceNode.getParallelism());
+ assertEquals(8, keyProjector.getParallelism());
+ assertEquals(8, sinkNode.getParallelism());
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
+ }
+ }
+}
[06/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
new file mode 100644
index 0000000..e65758f
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.junit.Test;
+
+@SuppressWarnings({"serial", "unchecked"})
+public class NestedIterationsTest extends CompilerTestBase {
+
+ @Test
+ public void testRejectNestedBulkIterations() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> data = env.generateSequence(1, 100);
+
+ IterativeDataSet<Long> outerIteration = data.iterate(100);
+
+ IterativeDataSet<Long> innerIteration = outerIteration.map(new IdentityMapper<Long>()).iterate(100);
+
+ DataSet<Long> innerResult = innerIteration.closeWith(innerIteration.map(new IdentityMapper<Long>()));
+
+ DataSet<Long> outerResult = outerIteration.closeWith(innerResult.map(new IdentityMapper<Long>()));
+
+ outerResult.print();
+
+ Plan p = env.createProgramPlan();
+
+ try {
+ compileNoStats(p);
+ }
+ catch (CompilerException e) {
+ assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRejectNestedWorksetIterations() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> outerIteration = data.iterateDelta(data, 100, 0);
+
+ DataSet<Tuple2<Long, Long>> inOuter = outerIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> innerIteration = inOuter.iterateDelta(inOuter, 100, 0);
+
+ DataSet<Tuple2<Long, Long>> inInner = innerIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> innerResult = innerIteration.closeWith(inInner, inInner).map(new IdentityMapper<Tuple2<Long,Long>>());
+
+ DataSet<Tuple2<Long, Long>> outerResult = outerIteration.closeWith(innerResult, innerResult);
+
+ outerResult.print();
+
+ Plan p = env.createProgramPlan();
+
+ try {
+ compileNoStats(p);
+ }
+ catch (CompilerException e) {
+ assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBulkIterationInClosure() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Long> data1 = env.generateSequence(1, 100);
+ DataSet<Long> data2 = env.generateSequence(1, 100);
+
+ IterativeDataSet<Long> firstIteration = data1.iterate(100);
+
+ DataSet<Long> firstResult = firstIteration.closeWith(firstIteration.map(new IdentityMapper<Long>()));
+
+
+ IterativeDataSet<Long> mainIteration = data2.map(new IdentityMapper<Long>()).iterate(100);
+
+ DataSet<Long> joined = mainIteration.join(firstResult)
+ .where(new IdentityKeyExtractor<Long>()).equalTo(new IdentityKeyExtractor<Long>())
+ .with(new DummyFlatJoinFunction<Long>());
+
+ DataSet<Long> mainResult = mainIteration.closeWith(joined);
+
+ mainResult.print();
+
+ Plan p = env.createProgramPlan();
+
+ // optimizer should be able to translate this
+ OptimizedPlan op = compileNoStats(p);
+
+ // job graph generator should be able to translate this
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testDeltaIterationInClosure() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Long, Long>> data1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+ DataSet<Tuple2<Long, Long>> data2 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> firstIteration = data1.iterateDelta(data1, 100, 0);
+
+ DataSet<Tuple2<Long, Long>> inFirst = firstIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> firstResult = firstIteration.closeWith(inFirst, inFirst).map(new IdentityMapper<Tuple2<Long,Long>>());
+
+
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> mainIteration = data2.iterateDelta(data2, 100, 0);
+
+ DataSet<Tuple2<Long, Long>> joined = mainIteration.getWorkset().join(firstResult).where(0).equalTo(0)
+ .projectFirst(0).projectSecond(0);
+
+ DataSet<Tuple2<Long, Long>> mainResult = mainIteration.closeWith(joined, joined);
+
+ mainResult.print();
+
+ Plan p = env.createProgramPlan();
+
+ // optimizer should be able to translate this
+ OptimizedPlan op = compileNoStats(p);
+
+ // job graph generator should be able to translate this
+ new JobGraphGenerator().compileJobGraph(op);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
new file mode 100644
index 0000000..2b42f85
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class PartitionPushdownTest extends CompilerTestBase {
+
+ @Test
+ public void testPartitioningNotPushedDown() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input
+ .groupBy(0, 1).sum(2)
+ .groupBy(0).sum(1)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+
+ SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
+ SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
+
+ assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy());
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPartitioningReused() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
+
+ input
+ .groupBy(0).sum(1)
+ .groupBy(0, 1).sum(2)
+ .print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+
+ SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
+
+ assertEquals(ShipStrategyType.FORWARD, agg2Reducer.getInput().getShipStrategy());
+
+ assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
+ assertEquals(new FieldList(0), agg1Reducer.getInput().getShipStrategyKeys());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
new file mode 100644
index 0000000..16684dc
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
@@ -0,0 +1,845 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.translation.JavaPlan;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@SuppressWarnings("serial")
+public class PartitioningReusageTest extends CompilerTestBase {
+
+ @Test
+ public void noPreviousPartitioningJoin1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(0,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+
+ }
+
+ @Test
+ public void noPreviousPartitioningJoin2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+
+ }
+
+ @Test
+ public void reuseSinglePartitioningJoin1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(0,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+
+ }
+
+ @Test
+ public void reuseSinglePartitioningJoin2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseSinglePartitioningJoin3() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .join(set2.partitionByHash(2, 1)
+ .map(new MockMapper())
+ .withForwardedFields("2;1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseSinglePartitioningJoin4() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0)
+ .map(new MockMapper()).withForwardedFields("0")
+ .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseSinglePartitioningJoin5() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .join(set2.partitionByHash(2)
+ .map(new MockMapper())
+ .withForwardedFields("2"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .join(set2.partitionByHash(0,1)
+ .map(new MockMapper())
+ .withForwardedFields("0;1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(0,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+
+ @Test
+ public void reuseBothPartitioningJoin2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .join(set2.partitionByHash(1,2)
+ .map(new MockMapper())
+ .withForwardedFields("1;2"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin3() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0)
+ .map(new MockMapper()).withForwardedFields("0")
+ .join(set2.partitionByHash(2,1)
+ .map(new MockMapper())
+ .withForwardedFields("2;1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,1).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin4() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0,2)
+ .map(new MockMapper()).withForwardedFields("0;2")
+ .join(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,2).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin5() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(2)
+ .map(new MockMapper()).withForwardedFields("2")
+ .join(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,2).equalTo(2,1).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin6() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(0)
+ .map(new MockMapper()).withForwardedFields("0")
+ .join(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,2).equalTo(1,2).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+ @Test
+ public void reuseBothPartitioningJoin7() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
+ .partitionByHash(2)
+ .map(new MockMapper()).withForwardedFields("2")
+ .join(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"),
+ JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
+ .where(0,2).equalTo(1,2).with(new MockJoin());
+
+ joined.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidJoinInputProperties(join);
+ }
+
+
+ @Test
+ public void noPreviousPartitioningCoGroup1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .coGroup(set2)
+ .where(0,1).equalTo(0,1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+
+ }
+
+ @Test
+ public void noPreviousPartitioningCoGroup2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .coGroup(set2)
+ .where(0,1).equalTo(2,1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+
+ }
+
+ @Test
+ public void reuseSinglePartitioningCoGroup1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .coGroup(set2)
+ .where(0,1).equalTo(0,1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+
+ }
+
+ @Test
+ public void reuseSinglePartitioningCoGroup2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .coGroup(set2)
+ .where(0,1).equalTo(2,1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseSinglePartitioningCoGroup3() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .coGroup(set2.partitionByHash(2, 1)
+ .map(new MockMapper())
+ .withForwardedFields("2;1"))
+ .where(0,1).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseSinglePartitioningCoGroup4() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0)
+ .map(new MockMapper()).withForwardedFields("0")
+ .coGroup(set2)
+ .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseSinglePartitioningCoGroup5() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .coGroup(set2.partitionByHash(2)
+ .map(new MockMapper())
+ .withForwardedFields("2"))
+ .where(0,1).equalTo(2,1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup1() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .coGroup(set2.partitionByHash(0, 1)
+ .map(new MockMapper())
+ .withForwardedFields("0;1"))
+ .where(0, 1).equalTo(0, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+
+ @Test
+ public void reuseBothPartitioningCoGroup2() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0,1)
+ .map(new MockMapper()).withForwardedFields("0;1")
+ .coGroup(set2.partitionByHash(1, 2)
+ .map(new MockMapper())
+ .withForwardedFields("1;2"))
+ .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup3() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0)
+ .map(new MockMapper()).withForwardedFields("0")
+ .coGroup(set2.partitionByHash(2, 1)
+ .map(new MockMapper())
+ .withForwardedFields("2;1"))
+ .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup4() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(0,2)
+ .map(new MockMapper()).withForwardedFields("0;2")
+ .coGroup(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"))
+ .where(0, 2).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup5() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(2)
+ .map(new MockMapper()).withForwardedFields("2")
+ .coGroup(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"))
+ .where(0, 2).equalTo(2, 1).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup6() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(2)
+ .map(new MockMapper()).withForwardedFields("2")
+ .coGroup(set2.partitionByHash(2)
+ .map(new MockMapper())
+ .withForwardedFields("2"))
+ .where(0, 2).equalTo(1, 2).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+ @Test
+ public void reuseBothPartitioningCoGroup7() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
+ .partitionByHash(2)
+ .map(new MockMapper()).withForwardedFields("2")
+ .coGroup(set2.partitionByHash(1)
+ .map(new MockMapper())
+ .withForwardedFields("1"))
+ .where(0, 2).equalTo(1, 2).with(new MockCoGroup());
+
+ coGrouped.print();
+ JavaPlan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+
+ checkValidCoGroupInputProperties(coGroup);
+ }
+
+
+
+ private void checkValidJoinInputProperties(DualInputPlanNode join) {
+
+ GlobalProperties inProps1 = join.getInput1().getGlobalProperties();
+ GlobalProperties inProps2 = join.getInput2().getGlobalProperties();
+
+ if(inProps1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ inProps2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
+
+ // check that both inputs are hash partitioned on the same fields
+ FieldList pFields1 = inProps1.getPartitioningFields();
+ FieldList pFields2 = inProps2.getPartitioningFields();
+
+ assertTrue("Inputs are not the same number of fields. Input 1: "+pFields1+", Input 2: "+pFields2,
+ pFields1.size() == pFields2.size());
+
+ FieldList reqPFields1 = join.getKeysForInput1();
+ FieldList reqPFields2 = join.getKeysForInput2();
+
+ for(int i=0; i<pFields1.size(); i++) {
+
+ // get fields
+ int f1 = pFields1.get(i);
+ int f2 = pFields2.get(i);
+
+ // check that field positions in original key field list are identical
+ int pos1 = getPosInFieldList(f1, reqPFields1);
+ int pos2 = getPosInFieldList(f2, reqPFields2);
+
+ if(pos1 < 0) {
+ fail("Input 1 is partitioned on field "+f1+" which is not contained in the key set "+reqPFields1);
+ }
+ if(pos2 < 0) {
+ fail("Input 2 is partitioned on field "+f2+" which is not contained in the key set "+reqPFields2);
+ }
+ if(pos1 != pos2) {
+ fail("Inputs are not partitioned on the same key fields");
+ }
+ }
+
+ }
+ else if(inProps1.getPartitioning() == PartitioningProperty.FULL_REPLICATION &&
+ inProps2.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED) {
+ // we are good. No need to check for fields
+ }
+ else if(inProps1.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED &&
+ inProps2.getPartitioning() == PartitioningProperty.FULL_REPLICATION) {
+ // we are good. No need to check for fields
+ }
+ else {
+ throw new UnsupportedOperationException("This method has only been implemented to check for hash partitioned coGroupinputs");
+ }
+
+ }
+
+ private void checkValidCoGroupInputProperties(DualInputPlanNode coGroup) {
+
+ GlobalProperties inProps1 = coGroup.getInput1().getGlobalProperties();
+ GlobalProperties inProps2 = coGroup.getInput2().getGlobalProperties();
+
+ if(inProps1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ inProps2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
+
+ // check that both inputs are hash partitioned on the same fields
+ FieldList pFields1 = inProps1.getPartitioningFields();
+ FieldList pFields2 = inProps2.getPartitioningFields();
+
+ assertTrue("Inputs are not the same number of fields. Input 1: "+pFields1+", Input 2: "+pFields2,
+ pFields1.size() == pFields2.size());
+
+ FieldList reqPFields1 = coGroup.getKeysForInput1();
+ FieldList reqPFields2 = coGroup.getKeysForInput2();
+
+ for(int i=0; i<pFields1.size(); i++) {
+
+ // get fields
+ int f1 = pFields1.get(i);
+ int f2 = pFields2.get(i);
+
+ // check that field positions in original key field list are identical
+ int pos1 = getPosInFieldList(f1, reqPFields1);
+ int pos2 = getPosInFieldList(f2, reqPFields2);
+
+ if(pos1 < 0) {
+ fail("Input 1 is partitioned on field "+f1+" which is not contained in the key set "+reqPFields1);
+ }
+ if(pos2 < 0) {
+ fail("Input 2 is partitioned on field "+f2+" which is not contained in the key set "+reqPFields2);
+ }
+ if(pos1 != pos2) {
+ fail("Inputs are not partitioned on the same key fields");
+ }
+ }
+
+ }
+ else {
+ throw new UnsupportedOperationException("This method has only been implemented to check for hash partitioned coGroup inputs");
+ }
+
+ }
+
+ private int getPosInFieldList(int field, FieldList list) {
+
+ int pos;
+ for(pos=0; pos<list.size(); pos++) {
+ if(field == list.get(pos)) {
+ break;
+ }
+ }
+ if(pos == list.size()) {
+ return -1;
+ } else {
+ return pos;
+ }
+
+ }
+
+
+
+ public static class MockMapper implements MapFunction<Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+ @Override
+ public Tuple3<Integer, Integer, Integer> map(Tuple3<Integer, Integer, Integer> value) throws Exception {
+ return null;
+ }
+ }
+
+ public static class MockJoin implements JoinFunction<Tuple3<Integer, Integer, Integer>,
+ Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+ @Override
+ public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
+ return null;
+ }
+ }
+
+ public static class MockCoGroup implements CoGroupFunction<Tuple3<Integer, Integer, Integer>,
+ Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+ @Override
+ public void coGroup(Iterable<Tuple3<Integer, Integer, Integer>> first, Iterable<Tuple3<Integer, Integer, Integer>> second,
+ Collector<Tuple3<Integer, Integer, Integer>> out) throws Exception {
+
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
new file mode 100644
index 0000000..86f01b0
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.configuration.Configuration;
+
+@SuppressWarnings("serial")
+public class PipelineBreakerTest extends CompilerTestBase {
+
+ @Test
+ public void testPipelineBreakerWithBroadcastVariable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
+
+ DataSet<Long> result = source.map(new IdentityMapper<Long>())
+ .map(new IdentityMapper<Long>())
+ .withBroadcastSet(source, "bc");
+
+ result.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput().getTempMode().breaksPipeline());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPipelineBreakerBroadcastedAllReduce() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
+
+ DataSet<Long> bcInput1 = sourceWithMapper
+ .map(new IdentityMapper<Long>())
+ .reduce(new SelectOneReducer<Long>());
+ DataSet<Long> bcInput2 = env.generateSequence(1, 10);
+
+ DataSet<Long> result = sourceWithMapper
+ .map(new IdentityMapper<Long>())
+ .withBroadcastSet(bcInput1, "bc1")
+ .withBroadcastSet(bcInput2, "bc2");
+
+ result.print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput().getTempMode().breaksPipeline());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPipelineBreakerBroadcastedPartialSolution() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+
+ DataSet<Long> initialSource = env.generateSequence(1, 10);
+ IterativeDataSet<Long> iteration = initialSource.iterate(100);
+
+
+ DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
+
+ DataSet<Long> bcInput1 = sourceWithMapper
+ .map(new IdentityMapper<Long>())
+ .reduce(new SelectOneReducer<Long>());
+
+ DataSet<Long> result = sourceWithMapper
+ .map(new IdentityMapper<Long>())
+ .withBroadcastSet(iteration, "bc2")
+ .withBroadcastSet(bcInput1, "bc1");
+
+
+ iteration.closeWith(result).print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ BulkIterationPlanNode iterationPlanNode = (BulkIterationPlanNode) sink.getInput().getSource();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) iterationPlanNode.getRootOfStepFunction();
+
+ assertTrue(mapper.getInput().getTempMode().breaksPipeline());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testPilelineBreakerWithCross() {
+ try {
+ {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> initialSource = env.generateSequence(1, 10);
+
+ Configuration conf= new Configuration();
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST);
+ initialSource
+ .map(new IdentityMapper<Long>())
+ .cross(initialSource).withParameters(conf)
+ .print();
+
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
+ }
+
+ {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> initialSource = env.generateSequence(1, 10);
+
+ Configuration conf= new Configuration();
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
+ initialSource
+ .map(new IdentityMapper<Long>())
+ .cross(initialSource).withParameters(conf)
+ .print();
+
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
+ }
+
+ {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> initialSource = env.generateSequence(1, 10);
+
+ Configuration conf= new Configuration();
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
+ initialSource
+ .map(new IdentityMapper<Long>())
+ .cross(initialSource).withParameters(conf)
+ .print();
+
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
+ }
+
+ {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(64);
+
+ DataSet<Long> initialSource = env.generateSequence(1, 10);
+
+ Configuration conf= new Configuration();
+ conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
+ initialSource
+ .map(new IdentityMapper<Long>())
+ .cross(initialSource).withParameters(conf)
+ .print();
+
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
+
+ assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
new file mode 100644
index 0000000..7be2b16
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
@@ -0,0 +1,897 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.DataSource;
+import org.apache.flink.api.java.operators.translation.JavaPlan;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@SuppressWarnings({"serial"})
+public class PropertyDataSourceTest extends CompilerTestBase {
+
+ private List<Tuple3<Long, SomePojo, String>> tuple3PojoData = new ArrayList<Tuple3<Long, SomePojo, String>>();
+ private TupleTypeInfo<Tuple3<Long, SomePojo, String>> tuple3PojoType = new TupleTypeInfo<Tuple3<Long, SomePojo, String>>(
+ BasicTypeInfo.LONG_TYPE_INFO,
+ TypeExtractor.createTypeInfo(SomePojo.class),
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
+ @Test
+ public void checkSinglePartitionedSource1() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource2() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(1, 0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource3() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("*");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1, 2, 3, 4)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource4() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1, 2, 3)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource5() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.stringField");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(3)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource6() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.intField; f2");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2, 4)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedSource7() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("byDate", 1, 0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
+ Assert.assertTrue(gprops.getCustomPartitioner() != null);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedGroupedSource1() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0)
+ .splitsGroupedBy(0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedGroupedSource2() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0)
+ .splitsGroupedBy(1, 0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0, 1)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedGroupedSource3() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(1)
+ .splitsGroupedBy(0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedGroupedSource4() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0, 1)
+ .splitsGroupedBy(0);
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedGroupedSource5() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f2")
+ .splitsGroupedBy("f2");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(4)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(4)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedGroupedSource6() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.intField")
+ .splitsGroupedBy("f0; f1.intField");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0,2)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedGroupedSource7() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.intField")
+ .splitsGroupedBy("f1");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedGroupedSource8() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1")
+ .splitsGroupedBy("f1.stringField");
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedOrderedSource1() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(1)
+ .splitsOrderedBy(new int[]{1}, new Order[]{Order.ASCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedOrderedSource2() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(1)
+ .splitsOrderedBy(new int[]{1, 0}, new Order[]{Order.ASCENDING, Order.DESCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1, 0)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedOrderedSource3() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0)
+ .splitsOrderedBy(new int[]{1}, new Order[]{Order.ASCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedOrderedSource4() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy(0, 1)
+ .splitsOrderedBy(new int[]{1}, new Order[]{Order.DESCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedOrderedSource5() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.intField")
+ .splitsOrderedBy("f0; f1.intField", new Order[]{Order.ASCENDING, Order.DESCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0,2)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkSinglePartitionedOrderedSource6() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1.intField")
+ .splitsOrderedBy("f1", new Order[]{Order.DESCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3)));
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+ @Test
+ public void checkSinglePartitionedOrderedSource7() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
+
+ data.getSplitDataProperties()
+ .splitsPartitionedBy("f1")
+ .splitsOrderedBy("f1.stringField", new Order[]{Order.ASCENDING});
+
+ data.print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
+
+ GlobalProperties gprops = sourceNode.getGlobalProperties();
+ LocalProperties lprops = sourceNode.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
+ Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
+ Assert.assertTrue(lprops.getGroupedFields() == null);
+ Assert.assertTrue(lprops.getOrdering() == null);
+
+ }
+
+
+ @Test
+ public void checkCoPartitionedSources1() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data1 =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data1.getSplitDataProperties()
+ .splitsPartitionedBy("byDate", 0);
+
+ DataSource<Tuple2<Long, String>> data2 =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data2.getSplitDataProperties()
+ .splitsPartitionedBy("byDate", 0);
+
+ data1.union(data2).print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
+ SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();
+
+ GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
+ LocalProperties lprops1 = sourceNode1.getLocalProperties();
+ GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
+ LocalProperties lprops2 = sourceNode2.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
+ Assert.assertTrue(lprops1.getGroupedFields() == null);
+ Assert.assertTrue(lprops1.getOrdering() == null);
+
+ Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
+ Assert.assertTrue(lprops2.getGroupedFields() == null);
+ Assert.assertTrue(lprops2.getOrdering() == null);
+
+ Assert.assertTrue(gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
+ }
+
+ @Test
+ public void checkCoPartitionedSources2() {
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSource<Tuple2<Long, String>> data1 =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data1.getSplitDataProperties()
+ .splitsPartitionedBy("byCountry", 0);
+
+ DataSource<Tuple2<Long, String>> data2 =
+ env.readCsvFile("/some/path").types(Long.class, String.class);
+
+ data2.getSplitDataProperties()
+ .splitsPartitionedBy("byDate", 0);
+
+ data1.union(data2).print();
+
+ JavaPlan plan = env.createProgramPlan();
+
+ // submit the plan to the compiler
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // check the optimized Plan
+ SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
+ SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
+ SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();
+
+ GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
+ LocalProperties lprops1 = sourceNode1.getLocalProperties();
+ GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
+ LocalProperties lprops2 = sourceNode2.getLocalProperties();
+
+ Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
+ Assert.assertTrue(lprops1.getGroupedFields() == null);
+ Assert.assertTrue(lprops1.getOrdering() == null);
+
+ Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
+ Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
+ Assert.assertTrue(lprops2.getGroupedFields() == null);
+ Assert.assertTrue(lprops2.getOrdering() == null);
+
+ Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
+ }
+
+
+ public static class SomePojo {
+ public double doubleField;
+ public int intField;
+ public String stringField;
+ }
+
+}
+
+
[19/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
new file mode 100644
index 0000000..55b8785
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dag.WorksetIterationNode.SingleRootJoiner;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.NoOpDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * A node in the optimizer's program representation for a bulk iteration.
+ */
+public class BulkIterationNode extends SingleInputNode implements IterationNode {
+
+ private BulkPartialSolutionNode partialSolution;
+
+ private OptimizerNode terminationCriterion;
+
+ private OptimizerNode nextPartialSolution;
+
+ private DagConnection rootConnection; // connection out of the next partial solution
+
+ private DagConnection terminationCriterionRootConnection; // connection out of the term. criterion
+
+ private OptimizerNode singleRoot;
+
+ private final int costWeight;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new node for the bulk iteration.
+ *
+ * @param iteration The bulk iteration the node represents.
+ */
+ public BulkIterationNode(BulkIterationBase<?> iteration) {
+ super(iteration);
+
+ if (iteration.getMaximumNumberOfIterations() <= 0) {
+ throw new CompilerException("BulkIteration must have a maximum number of iterations specified.");
+ }
+
+ int numIters = iteration.getMaximumNumberOfIterations();
+
+ this.costWeight = (numIters > 0 && numIters < OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) ?
+ numIters : OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public BulkIterationBase<?> getIterationContract() {
+ return (BulkIterationBase<?>) getOperator();
+ }
+
+ /**
+ * Gets the partialSolution from this BulkIterationNode.
+ *
+ * @return The partialSolution.
+ */
+ public BulkPartialSolutionNode getPartialSolution() {
+ return partialSolution;
+ }
+
+ /**
+ * Sets the partialSolution for this BulkIterationNode.
+ *
+ * @param partialSolution The partialSolution to set.
+ */
+ public void setPartialSolution(BulkPartialSolutionNode partialSolution) {
+ this.partialSolution = partialSolution;
+ }
+
+
+ /**
+ * Gets the nextPartialSolution from this BulkIterationNode.
+ *
+ * @return The nextPartialSolution.
+ */
+ public OptimizerNode getNextPartialSolution() {
+ return nextPartialSolution;
+ }
+
+ /**
+ * Sets the nextPartialSolution for this BulkIterationNode.
+ *
+ * @param nextPartialSolution The nextPartialSolution to set.
+ */
+ public void setNextPartialSolution(OptimizerNode nextPartialSolution, OptimizerNode terminationCriterion) {
+
+ // check if the root of the step function has the same DOP as the iteration
+ // or if the step function has any operator at all
+ if (nextPartialSolution.getParallelism() != getParallelism() ||
+ nextPartialSolution == partialSolution || nextPartialSolution instanceof BinaryUnionNode)
+ {
+ // add a no-op to the root to express the re-partitioning
+ NoOpNode noop = new NoOpNode();
+ noop.setDegreeOfParallelism(getParallelism());
+
+ DagConnection noOpConn = new DagConnection(nextPartialSolution, noop, ExecutionMode.PIPELINED);
+ noop.setIncomingConnection(noOpConn);
+ nextPartialSolution.addOutgoingConnection(noOpConn);
+
+ nextPartialSolution = noop;
+ }
+
+ this.nextPartialSolution = nextPartialSolution;
+ this.terminationCriterion = terminationCriterion;
+
+ if (terminationCriterion == null) {
+ this.singleRoot = nextPartialSolution;
+ this.rootConnection = new DagConnection(nextPartialSolution, ExecutionMode.PIPELINED);
+ }
+ else {
+ // we have a termination criterion
+ SingleRootJoiner singleRootJoiner = new SingleRootJoiner();
+ this.rootConnection = new DagConnection(nextPartialSolution, singleRootJoiner, ExecutionMode.PIPELINED);
+ this.terminationCriterionRootConnection = new DagConnection(terminationCriterion, singleRootJoiner,
+ ExecutionMode.PIPELINED);
+
+ singleRootJoiner.setInputs(this.rootConnection, this.terminationCriterionRootConnection);
+
+ this.singleRoot = singleRootJoiner;
+
+ // add connection to terminationCriterion for interesting properties visitor
+ terminationCriterion.addOutgoingConnection(terminationCriterionRootConnection);
+
+ }
+
+ nextPartialSolution.addOutgoingConnection(rootConnection);
+ }
+
+ public int getCostWeight() {
+ return this.costWeight;
+ }
+
+ public OptimizerNode getSingleRootOfStepFunction() {
+ return this.singleRoot;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public String getName() {
+ return "Bulk Iteration";
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new EmptySemanticProperties();
+ }
+
+ protected void readStubAnnotations() {}
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Properties and Optimization
+ // --------------------------------------------------------------------------------------------
+
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return Collections.<OperatorDescriptorSingle>singletonList(new NoOpDescriptor());
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ final InterestingProperties intProps = getInterestingProperties().clone();
+
+ if (this.terminationCriterion != null) {
+ // first propagate through termination Criterion. since it has no successors, it has no
+ // interesting properties
+ this.terminationCriterionRootConnection.setInterestingProperties(new InterestingProperties());
+ this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator));
+ }
+
+ // we need to make 2 interesting property passes, because the root of the step function needs also
+ // the interesting properties as generated by the partial solution
+
+ // give our own interesting properties (as generated by the iterations successors) to the step function and
+ // make the first pass
+ this.rootConnection.setInterestingProperties(intProps);
+ this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
+
+ // take the interesting properties of the partial solution and add them to the root interesting properties
+ InterestingProperties partialSolutionIntProps = this.partialSolution.getInterestingProperties();
+ intProps.getGlobalProperties().addAll(partialSolutionIntProps.getGlobalProperties());
+ intProps.getLocalProperties().addAll(partialSolutionIntProps.getLocalProperties());
+
+ // clear all interesting properties to prepare the second traversal
+ // this clears only the path down from the next partial solution. The paths down
+ // from the termination criterion (before they meet the paths down from the next partial solution)
+ // remain unaffected by this step
+ this.rootConnection.clearInterestingProperties();
+ this.nextPartialSolution.accept(InterestingPropertiesClearer.INSTANCE);
+
+ // 2nd pass
+ this.rootConnection.setInterestingProperties(intProps);
+ this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
+
+ // now add the interesting properties of the partial solution to the input
+ final InterestingProperties inProps = this.partialSolution.getInterestingProperties().clone();
+ inProps.addGlobalProperties(new RequestedGlobalProperties());
+ inProps.addLocalProperties(new RequestedLocalProperties());
+ this.inConn.setInterestingProperties(inProps);
+ }
+
+ @Override
+ public void clearInterestingProperties() {
+ super.clearInterestingProperties();
+
+ this.singleRoot.accept(InterestingPropertiesClearer.INSTANCE);
+ this.rootConnection.clearInterestingProperties();
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ // the resulting branches are those of the step function
+ // because the BulkPartialSolution takes the input's branches
+ addClosedBranches(getSingleRootOfStepFunction().closedBranchingNodes);
+ List<UnclosedBranchDescriptor> result = getSingleRootOfStepFunction().openBranches;
+
+ this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
+ }
+
+
+ @Override
+ protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
+ List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
+ {
+ // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
+ // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
+ // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
+ // we have a fitting candidate for the step function (often, work is pushed out of the step function).
+ // Among the candidates of the step function, we keep only those that meet the requested properties of the
+ // current candidate initial partial solution. That makes sure these properties exist at the beginning of
+ // the successive iteration.
+
+ // 1) Because we enumerate multiple times, we may need to clean the cached plans
+ // before starting another enumeration
+ this.nextPartialSolution.accept(PlanCacheCleaner.INSTANCE);
+ if (this.terminationCriterion != null) {
+ this.terminationCriterion.accept(PlanCacheCleaner.INSTANCE);
+ }
+
+ // 2) Give the partial solution the properties of the current candidate for the initial partial solution
+ this.partialSolution.setCandidateProperties(in.getGlobalProperties(), in.getLocalProperties(), in);
+ final BulkPartialSolutionPlanNode pspn = this.partialSolution.getCurrentPartialSolutionPlanNode();
+
+ // 3) Get the alternative plans
+ List<PlanNode> candidates = this.nextPartialSolution.getAlternativePlans(estimator);
+
+ // 4) Make sure that the beginning of the step function does not assume properties that
+ // are not also produced by the end of the step function.
+
+ {
+ List<PlanNode> newCandidates = new ArrayList<PlanNode>();
+
+ for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
+ PlanNode candidate = planDeleter.next();
+
+ GlobalProperties atEndGlobal = candidate.getGlobalProperties();
+ LocalProperties atEndLocal = candidate.getLocalProperties();
+
+ FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobal, atEndLocal);
+ if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
+ ; // depends only through broadcast variable on the partial solution
+ }
+ else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
+ // attach a no-op node through which we create the properties of the original input
+ Channel toNoOp = new Channel(candidate);
+ globPropsReq.parameterizeChannel(toNoOp, false, rootConnection.getDataExchangeMode(), false);
+ locPropsReq.parameterizeChannel(toNoOp);
+
+ UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
+ rebuildPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
+
+ SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
+ rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
+ estimator.costOperator(rebuildPropertiesPlanNode);
+
+ GlobalProperties atEndGlobalModified = rebuildPropertiesPlanNode.getGlobalProperties();
+ LocalProperties atEndLocalModified = rebuildPropertiesPlanNode.getLocalProperties();
+
+ if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
+ FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobalModified, atEndLocalModified);
+
+ if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
+ newCandidates.add(rebuildPropertiesPlanNode);
+ }
+ }
+
+ planDeleter.remove();
+ }
+ }
+ }
+
+ if (candidates.isEmpty()) {
+ return;
+ }
+
+ // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
+ if (terminationCriterion == null) {
+ for (PlanNode candidate : candidates) {
+ BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate);
+ GlobalProperties gProps = candidate.getGlobalProperties().clone();
+ LocalProperties lProps = candidate.getLocalProperties().clone();
+ node.initProperties(gProps, lProps);
+ target.add(node);
+ }
+ }
+ else if (candidates.size() > 0) {
+ List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);
+
+ SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
+
+ for (PlanNode candidate : candidates) {
+ for (PlanNode terminationCandidate : terminationCriterionCandidates) {
+ if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
+ BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate);
+ GlobalProperties gProps = candidate.getGlobalProperties().clone();
+ LocalProperties lProps = candidate.getLocalProperties().clone();
+ node.initProperties(gProps, lProps);
+ target.add(node);
+
+ }
+ }
+ }
+
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Iteration Specific Traversals
+ // --------------------------------------------------------------------------------------------
+
+ public void acceptForStepFunction(Visitor<OptimizerNode> visitor) {
+ this.singleRoot.accept(visitor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
new file mode 100644
index 0000000..25a7eef
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.BulkIterationBase.PartialSolutionPlaceHolder;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.PlanNode;
+
+/**
+ * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
+ */
+public class BulkPartialSolutionNode extends AbstractPartialSolutionNode {
+
+ private final BulkIterationNode iterationNode;
+
+
+ public BulkPartialSolutionNode(PartialSolutionPlaceHolder<?> psph, BulkIterationNode iterationNode) {
+ super(psph);
+ this.iterationNode = iterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
+ if (this.cachedPlans != null) {
+ throw new IllegalStateException();
+ } else {
+ this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
+ "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
+ }
+ }
+
+ public BulkPartialSolutionPlanNode getCurrentPartialSolutionPlanNode() {
+ if (this.cachedPlans != null) {
+ return (BulkPartialSolutionPlanNode) this.cachedPlans.get(0);
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ public BulkIterationNode getIterationNode() {
+ return this.iterationNode;
+ }
+
+ @Override
+ public void computeOutputEstimates(DataStatistics statistics) {
+ copyEstimates(this.iterationNode.getPredecessorNode());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the operator (here the {@link PartialSolutionPlaceHolder}) that is represented by this
+ * optimizer node.
+ *
+ * @return The operator represented by this optimizer node.
+ */
+ @Override
+ public PartialSolutionPlaceHolder<?> getOperator() {
+ return (PartialSolutionPlaceHolder<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Bulk Partial Solution";
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ OptimizerNode inputToIteration = this.iterationNode.getPredecessorNode();
+
+ addClosedBranches(inputToIteration.closedBranchingNodes);
+ List<UnclosedBranchDescriptor> fromInput = inputToIteration.getBranchesForParent(this.iterationNode.getIncomingConnection());
+ this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
new file mode 100644
index 0000000..92076c3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.CoGroupDescriptor;
+import org.apache.flink.optimizer.operators.CoGroupWithSolutionSetFirstDescriptor;
+import org.apache.flink.optimizer.operators.CoGroupWithSolutionSetSecondDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+
+/**
+ * The Optimizer representation of a <i>CoGroup</i> operator.
+ */
+public class CoGroupNode extends TwoInputNode {
+
+ private List<OperatorDescriptorDual> dataProperties;
+
+ public CoGroupNode(CoGroupOperatorBase<?, ?, ?, ?> operator) {
+ super(operator);
+ this.dataProperties = initializeDataProperties(operator.getCustomPartitioner());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the operator for this CoGroup node.
+ *
+ * @return The CoGroup operator.
+ */
+ @Override
+ public CoGroupOperatorBase<?, ?, ?, ?> getOperator() {
+ return (CoGroupOperatorBase<?, ?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "CoGroup";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return this.dataProperties;
+ }
+
+ public void makeCoGroupWithSolutionSet(int solutionsetInputIndex) {
+ OperatorDescriptorDual op;
+ if (solutionsetInputIndex == 0) {
+ op = new CoGroupWithSolutionSetFirstDescriptor(keys1, keys2);
+ } else if (solutionsetInputIndex == 1) {
+ op = new CoGroupWithSolutionSetSecondDescriptor(keys1, keys2);
+ } else {
+ throw new IllegalArgumentException();
+ }
+ this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(op);
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // for CoGroup, we currently make no reasonable default estimates
+ }
+
+ private List<OperatorDescriptorDual> initializeDataProperties(Partitioner<?> customPartitioner) {
+ Ordering groupOrder1 = null;
+ Ordering groupOrder2 = null;
+
+ CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
+ groupOrder1 = cgc.getGroupOrderForInputOne();
+ groupOrder2 = cgc.getGroupOrderForInputTwo();
+
+ if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
+ groupOrder1 = null;
+ }
+ if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
+ groupOrder2 = null;
+ }
+
+ CoGroupDescriptor descr = new CoGroupDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2);
+ if (customPartitioner != null) {
+ descr.setCustomPartitioner(customPartitioner);
+ }
+
+ return Collections.<OperatorDescriptorDual>singletonList(descr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
new file mode 100644
index 0000000..93be1e4
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CollectorMapNode.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.CollectorMapDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+/**
+ * The optimizer's internal representation of a <i>Map</i> operator node.
+ */
+public class CollectorMapNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+
+ public CollectorMapNode(SingleInputOperator<?, ?, ?> operator) {
+ super(operator);
+
+ this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new CollectorMapDescriptor());
+ }
+
+ @Override
+ public String getName() {
+ return "Map";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ /**
+ * Computes the estimates for the Map operator. Map takes one value and transforms it into another value.
+ * The cardinality consequently stays the same.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
new file mode 100644
index 0000000..8de67e8
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.CrossOperatorBase;
+import org.apache.flink.api.common.operators.base.CrossOperatorBase.CrossHint;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.operators.CrossBlockOuterFirstDescriptor;
+import org.apache.flink.optimizer.operators.CrossBlockOuterSecondDescriptor;
+import org.apache.flink.optimizer.operators.CrossStreamOuterFirstDescriptor;
+import org.apache.flink.optimizer.operators.CrossStreamOuterSecondDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * The Optimizer representation of a <i>Cross</i> (Cartesian product) operator.
+ */
+public class CrossNode extends TwoInputNode {
+
+ private final List<OperatorDescriptorDual> dataProperties;
+
+ /**
+ * Creates a new CrossNode for the given operator.
+ *
+ * @param operation The Cross operator object.
+ */
+ public CrossNode(CrossOperatorBase<?, ?, ?, ?> operation) {
+ super(operation);
+
+ Configuration conf = operation.getParameters();
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
+
+ CrossHint hint = operation.getCrossHint();
+
+ if (localStrategy != null) {
+
+ final boolean allowBCfirst = hint != CrossHint.SECOND_IS_SMALL;
+ final boolean allowBCsecond = hint != CrossHint.FIRST_IS_SMALL;
+
+ final OperatorDescriptorDual fixedDriverStrat;
+ if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
+ fixedDriverStrat = new CrossBlockOuterFirstDescriptor(allowBCfirst, allowBCsecond);
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND.equals(localStrategy)) {
+ fixedDriverStrat = new CrossBlockOuterSecondDescriptor(allowBCfirst, allowBCsecond);
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST.equals(localStrategy)) {
+ fixedDriverStrat = new CrossStreamOuterFirstDescriptor(allowBCfirst, allowBCsecond);
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND.equals(localStrategy)) {
+ fixedDriverStrat = new CrossStreamOuterSecondDescriptor(allowBCfirst, allowBCsecond);
+ } else {
+ throw new CompilerException("Invalid local strategy hint for cross contract: " + localStrategy);
+ }
+
+ this.dataProperties = Collections.singletonList(fixedDriverStrat);
+ }
+ else if (hint == CrossHint.SECOND_IS_SMALL) {
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+ list.add(new CrossBlockOuterSecondDescriptor(false, true));
+ list.add(new CrossStreamOuterFirstDescriptor(false, true));
+ this.dataProperties = list;
+ }
+ else if (hint == CrossHint.FIRST_IS_SMALL) {
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+ list.add(new CrossBlockOuterFirstDescriptor(true, false));
+ list.add(new CrossStreamOuterSecondDescriptor(true, false));
+ this.dataProperties = list;
+ }
+ else {
+ ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
+ list.add(new CrossBlockOuterFirstDescriptor());
+ list.add(new CrossBlockOuterSecondDescriptor());
+ list.add(new CrossStreamOuterFirstDescriptor());
+ list.add(new CrossStreamOuterSecondDescriptor());
+ this.dataProperties = list;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public CrossOperatorBase<?, ?, ?, ?> getOperator() {
+ return (CrossOperatorBase<?, ?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Cross";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return this.dataProperties;
+ }
+
+ /**
+ * We assume that the cardinality is the product of the input cardinalities
+ * and that the result width is the sum of the input widths.
+ *
+ * @param statistics The statistics object to optionally access.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ long card1 = getFirstPredecessorNode().getEstimatedNumRecords();
+ long card2 = getSecondPredecessorNode().getEstimatedNumRecords();
+ this.estimatedNumRecords = (card1 < 0 || card2 < 0) ? -1 : card1 * card2;
+
+ if (this.estimatedNumRecords >= 0) {
+ float width1 = getFirstPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width2 = getSecondPredecessorNode().getEstimatedAvgWidthPerOutputRecord();
+ float width = (width1 <= 0 || width2 <= 0) ? -1 : width1 + width2;
+
+ if (width > 0) {
+ this.estimatedOutputSize = (long) (width * this.estimatedNumRecords);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
new file mode 100644
index 0000000..4e65976
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.plandump.DumpableConnection;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+/**
+ * A connection between to operators. Represents an intermediate result
+ * and a data exchange between the two operators.
+ *
+ * The data exchange has a mode in which it performs (batch / pipelined).
+ *
+ * The data exchange strategy may be set on this connection, in which case
+ * it is fixed and will not be determined during candidate plan enumeration.
+ *
+ * During the enumeration of interesting properties, this connection also holds
+ * all interesting properties generated by the successor operator.
+ */
+public class DagConnection implements EstimateProvider, DumpableConnection<OptimizerNode> {
+
+ private final OptimizerNode source; // The source node of the connection
+
+ private final OptimizerNode target; // The target node of the connection.
+
+ private final ExecutionMode dataExchangeMode; // defines whether to use batch or pipelined data exchange
+
+ private InterestingProperties interestingProps; // local properties that succeeding nodes are interested in
+
+ private ShipStrategyType shipStrategy; // The data shipping strategy, if predefined.
+
+ private TempMode materializationMode = TempMode.NONE; // the materialization mode
+
+ private int maxDepth = -1;
+
+ private boolean breakPipeline; // whether this connection should break the pipeline due to potential deadlocks
+
+ /**
+ * Creates a new Connection between two nodes. The shipping strategy is by default <tt>NONE</tt>.
+ * The temp mode is by default <tt>NONE</tt>.
+ *
+ * @param source
+ * The source node.
+ * @param target
+ * The target node.
+ */
+ public DagConnection(OptimizerNode source, OptimizerNode target, ExecutionMode exchangeMode) {
+ this(source, target, null, exchangeMode);
+ }
+
+ /**
+ * Creates a new Connection between two nodes.
+ *
+ * @param source
+ * The source node.
+ * @param target
+ * The target node.
+ * @param shipStrategy
+ * The shipping strategy.
+ * @param exchangeMode
+ * The data exchange mode (pipelined / batch / batch only for shuffles / ... )
+ */
+ public DagConnection(OptimizerNode source, OptimizerNode target,
+ ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
+ {
+ if (source == null || target == null) {
+ throw new NullPointerException("Source and target must not be null.");
+ }
+ this.source = source;
+ this.target = target;
+ this.shipStrategy = shipStrategy;
+ this.dataExchangeMode = exchangeMode;
+ }
+
+ /**
+ * Constructor to create a result from an operator that is not
+ * consumed by another operator.
+ *
+ * @param source
+ * The source node.
+ */
+ public DagConnection(OptimizerNode source, ExecutionMode exchangeMode) {
+ if (source == null) {
+ throw new NullPointerException("Source and target must not be null.");
+ }
+ this.source = source;
+ this.target = null;
+ this.shipStrategy = ShipStrategyType.NONE;
+ this.dataExchangeMode = exchangeMode;
+ }
+
+ /**
+ * Gets the source of the connection.
+ *
+ * @return The source Node.
+ */
+ public OptimizerNode getSource() {
+ return this.source;
+ }
+
+ /**
+ * Gets the target of the connection.
+ *
+ * @return The target node.
+ */
+ public OptimizerNode getTarget() {
+ return this.target;
+ }
+
+ /**
+ * Gets the shipping strategy for this connection.
+ *
+ * @return The connection's shipping strategy.
+ */
+ public ShipStrategyType getShipStrategy() {
+ return this.shipStrategy;
+ }
+
+ /**
+ * Sets the shipping strategy for this connection.
+ *
+ * @param strategy
+ * The shipping strategy to be applied to this connection.
+ */
+ public void setShipStrategy(ShipStrategyType strategy) {
+ this.shipStrategy = strategy;
+ }
+
+ /**
+ * Gets the data exchange mode to use for this connection.
+ *
+ * @return The data exchange mode to use for this connection.
+ */
+ public ExecutionMode getDataExchangeMode() {
+ if (dataExchangeMode == null) {
+ throw new IllegalStateException("This connection does not have the data exchange mode set");
+ }
+ return dataExchangeMode;
+ }
+
+ /**
+ * Marks that this connection should do a decoupled data exchange (such as batched)
+ * rather then pipeline data. Connections are marked as pipeline breakers to avoid
+ * deadlock situations.
+ */
+ public void markBreaksPipeline() {
+ this.breakPipeline = true;
+ }
+
+ /**
+ * Checks whether this connection is marked to break the pipeline.
+ *
+ * @return True, if this connection is marked to break the pipeline, false otherwise.
+ */
+ public boolean isBreakingPipeline() {
+ return this.breakPipeline;
+ }
+
+ /**
+ * Gets the interesting properties object for this pact connection.
+ * If the interesting properties for this connections have not yet been set,
+ * this method returns null.
+ *
+ * @return The collection of all interesting properties, or null, if they have not yet been set.
+ */
+ public InterestingProperties getInterestingProperties() {
+ return this.interestingProps;
+ }
+
+ /**
+ * Sets the interesting properties for this pact connection.
+ *
+ * @param props The interesting properties.
+ */
+ public void setInterestingProperties(InterestingProperties props) {
+ if (this.interestingProps == null) {
+ this.interestingProps = props;
+ } else {
+ throw new IllegalStateException("Interesting Properties have already been set.");
+ }
+ }
+
+ public void clearInterestingProperties() {
+ this.interestingProps = null;
+ }
+
+ public void initMaxDepth() {
+
+ if (this.maxDepth == -1) {
+ this.maxDepth = this.source.getMaxDepth() + 1;
+ } else {
+ throw new IllegalStateException("Maximum path depth has already been initialized.");
+ }
+ }
+
+ public int getMaxDepth() {
+ if (this.maxDepth != -1) {
+ return this.maxDepth;
+ } else {
+ throw new IllegalStateException("Maximum path depth has not been initialized.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Estimates
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public long getEstimatedOutputSize() {
+ return this.source.getEstimatedOutputSize();
+ }
+
+ @Override
+ public long getEstimatedNumRecords() {
+ return this.source.getEstimatedNumRecords();
+ }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() {
+ return this.source.getEstimatedAvgWidthPerOutputRecord();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ public TempMode getMaterializationMode() {
+ return this.materializationMode;
+ }
+
+ public void setMaterializationMode(TempMode materializationMode) {
+ this.materializationMode = materializationMode;
+ }
+
+ public boolean isOnDynamicPath() {
+ return this.source.isOnDynamicPath();
+ }
+
+ public int getCostWeight() {
+ return this.source.getCostWeight();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder(50);
+ buf.append("Connection: ");
+
+ if (this.source == null) {
+ buf.append("null");
+ } else {
+ buf.append(this.source.getOperator().getName());
+ buf.append('(').append(this.source.getName()).append(')');
+ }
+
+ buf.append(" -> ");
+
+ if (this.shipStrategy != null) {
+ buf.append('[');
+ buf.append(this.shipStrategy.name());
+ buf.append(']').append(' ');
+ }
+
+ if (this.target == null) {
+ buf.append("null");
+ } else {
+ buf.append(this.target.getOperator().getName());
+ buf.append('(').append(this.target.getName()).append(')');
+ }
+
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
new file mode 100644
index 0000000..dbe04f4
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.operators.GenericDataSinkBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The Optimizer representation of a data sink.
+ */
+public class DataSinkNode extends OptimizerNode {
+
+ protected DagConnection input; // The input edge
+
+ /**
+ * Creates a new DataSinkNode for the given sink operator.
+ *
+ * @param sink The data sink contract object.
+ */
+ public DataSinkNode(GenericDataSinkBase<?> sink) {
+ super(sink);
+ }
+
+ // --------------------------------------------------------------------------------------
+
+ /**
+ * Gets the input of the sink.
+ *
+ * @return The input connection.
+ */
+ public DagConnection getInputConnection() {
+ return this.input;
+ }
+
+ /**
+ * Gets the predecessor of this node.
+ *
+ * @return The predecessor, or null, if no predecessor has been set.
+ */
+ public OptimizerNode getPredecessorNode() {
+ if(this.input != null) {
+ return input.getSource();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Gets the operator for which this optimizer sink node was created.
+ *
+ * @return The node's underlying operator.
+ */
+ @Override
+ public GenericDataSinkBase<?> getOperator() {
+ return (GenericDataSinkBase<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Data Sink";
+ }
+
+ @Override
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.singletonList(this.input);
+ }
+
+ /**
+ * Gets all outgoing connections, which is an empty set for the data sink.
+ *
+ * @return An empty list.
+ */
+ @Override
+ public List<DagConnection> getOutgoingConnections() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
+ Operator<?> children = getOperator().getInput();
+
+ final OptimizerNode pred;
+ final DagConnection conn;
+
+ pred = contractToNode.get(children);
+ conn = new DagConnection(pred, this, defaultExchangeMode);
+
+ // create the connection and add it
+ this.input = conn;
+ pred.addOutgoingConnection(conn);
+ }
+
+ /**
+ * Computes the estimated outputs for the data sink. Since the sink does not modify anything, it simply
+ * copies the output estimates from its direct predecessor.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ final InterestingProperties iProps = new InterestingProperties();
+
+ {
+ final Ordering partitioning = getOperator().getPartitionOrdering();
+ final DataDistribution dataDist = getOperator().getDataDistribution();
+ final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
+ if (partitioning != null) {
+ if(dataDist != null) {
+ partitioningProps.setRangePartitioned(partitioning, dataDist);
+ } else {
+ partitioningProps.setRangePartitioned(partitioning);
+ }
+ iProps.addGlobalProperties(partitioningProps);
+ }
+ iProps.addGlobalProperties(partitioningProps);
+ }
+
+ {
+ final Ordering localOrder = getOperator().getLocalOrder();
+ final RequestedLocalProperties orderProps = new RequestedLocalProperties();
+ if (localOrder != null) {
+ orderProps.setOrdering(localOrder);
+ }
+ iProps.addLocalProperties(orderProps);
+ }
+
+ this.input.setInterestingProperties(iProps);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Branch Handling
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ // we need to track open branches even in the sinks, because they get "closed" when
+ // we build a single "root" for the data flow plan
+ addClosedBranches(getPredecessorNode().closedBranchingNodes);
+ this.openBranches = getPredecessorNode().getBranchesForParent(this.input);
+ }
+
+ @Override
+ protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection parent) {
+ // return our own stack of open branches, because nothing is added
+ return this.openBranches;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Recursive Optimization
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ // check if we have a cached version
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ }
+
+ // calculate alternative sub-plans for predecessor
+ List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
+ List<PlanNode> outputPlans = new ArrayList<PlanNode>();
+
+ final int dop = getParallelism();
+ final int inDop = getPredecessorNode().getParallelism();
+
+ final ExecutionMode executionMode = this.input.getDataExchangeMode();
+ final boolean dopChange = dop != inDop;
+ final boolean breakPipeline = this.input.isBreakingPipeline();
+
+ InterestingProperties ips = this.input.getInterestingProperties();
+ for (PlanNode p : subPlans) {
+ for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
+ for (RequestedLocalProperties lp : ips.getLocalProperties()) {
+ Channel c = new Channel(p);
+ gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
+ lp.parameterizeChannel(c);
+ c.setRequiredLocalProps(lp);
+ c.setRequiredGlobalProps(gp);
+
+ // no need to check whether the created properties meet what we need in case
+ // of ordering or global ordering, because the only interesting properties we have
+ // are what we require
+ outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
+ }
+ }
+ }
+
+ // cost and prune the plans
+ for (PlanNode node : outputPlans) {
+ estimator.costOperator(node);
+ }
+ prunePlanAlternatives(outputPlans);
+
+ this.cachedPlans = outputPlans;
+ return outputPlans;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Function Annotation Handling
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new EmptySemanticProperties();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void accept(Visitor<OptimizerNode> visitor) {
+ if (visitor.preVisit(this)) {
+ if (getPredecessorNode() != null) {
+ getPredecessorNode().accept(visitor);
+ } else {
+ throw new CompilerException();
+ }
+ visitor.postVisit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
new file mode 100644
index 0000000..e4b35b7
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/DataSourceNode.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.io.FileInputFormat;
+import org.apache.flink.api.common.io.InputFormat;
+import org.apache.flink.api.common.io.NonParallelInput;
+import org.apache.flink.api.common.io.ReplicatingInputFormat;
+import org.apache.flink.api.common.io.statistics.BaseStatistics;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.GenericDataSourceBase.SplitDataProperties;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The optimizer's internal representation of a data source.
+ */
+public class DataSourceNode extends OptimizerNode {
+
+ private final boolean sequentialInput;
+
+ private final boolean replicatedInput;
+
+ private GlobalProperties gprops;
+
+ private LocalProperties lprops;
+
+ /**
+ * Creates a new DataSourceNode for the given contract.
+ *
+ * @param pactContract
+ * The data source contract object.
+ */
+ public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
+ super(pactContract);
+
+ if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
+ throw new IllegalArgumentException("Input format has not been set.");
+ }
+
+ if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
+ setDegreeOfParallelism(1);
+ this.sequentialInput = true;
+ } else {
+ this.sequentialInput = false;
+ }
+
+ this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
+ pactContract.getUserCodeWrapper().getUserCodeClass());
+
+ this.gprops = new GlobalProperties();
+ this.lprops = new LocalProperties();
+
+ SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();
+
+ if(replicatedInput) {
+ this.gprops.setFullyReplicated();
+ this.lprops = new LocalProperties();
+ } else if (splitProps != null) {
+ // configure data properties of data source using split properties
+ setDataPropertiesFromSplitProperties(splitProps);
+ }
+
+ }
+
+ /**
+ * Gets the contract object for this data source node.
+ *
+ * @return The contract.
+ */
+ @Override
+ public GenericDataSourceBase<?, ?> getOperator() {
+ return (GenericDataSourceBase<?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Data Source";
+ }
+
+ @Override
+ public void setDegreeOfParallelism(int degreeOfParallelism) {
+ // if unsplittable, parallelism remains at 1
+ if (!this.sequentialInput) {
+ super.setDegreeOfParallelism(degreeOfParallelism);
+ }
+ }
+
+ @Override
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.<DagConnection>emptyList();
+ }
+
+ @Override
+ public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultDataExchangeMode) {}
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // see, if we have a statistics object that can tell us a bit about the file
+ if (statistics != null) {
+ // instantiate the input format, as this is needed by the statistics
+ InputFormat<?, ?> format;
+ String inFormatDescription = "<unknown>";
+
+ try {
+ format = getOperator().getFormatWrapper().getUserCodeObject();
+ Configuration config = getOperator().getParameters();
+ format.configure(config);
+ }
+ catch (Throwable t) {
+ if (Optimizer.LOG.isWarnEnabled()) {
+ Optimizer.LOG.warn("Could not instantiate InputFormat to obtain statistics."
+ + " Limited statistics will be available.", t);
+ }
+ return;
+ }
+ try {
+ inFormatDescription = format.toString();
+ }
+ catch (Throwable t) {
+ // we can ignore this error, as it only prevents us to use a cosmetic string
+ }
+
+ // first of all, get the statistics from the cache
+ final String statisticsKey = getOperator().getStatisticsKey();
+ final BaseStatistics cachedStatistics = statistics.getBaseStatistics(statisticsKey);
+
+ BaseStatistics bs = null;
+ try {
+ bs = format.getStatistics(cachedStatistics);
+ }
+ catch (Throwable t) {
+ if (Optimizer.LOG.isWarnEnabled()) {
+ Optimizer.LOG.warn("Error obtaining statistics from input format: " + t.getMessage(), t);
+ }
+ }
+
+ if (bs != null) {
+ final long len = bs.getTotalInputSize();
+ if (len == BaseStatistics.SIZE_UNKNOWN) {
+ if (Optimizer.LOG.isInfoEnabled()) {
+ Optimizer.LOG.info("Compiler could not determine the size of input '" + inFormatDescription + "'. Using default estimates.");
+ }
+ }
+ else if (len >= 0) {
+ this.estimatedOutputSize = len;
+ }
+
+ final long card = bs.getNumberOfRecords();
+ if (card != BaseStatistics.NUM_RECORDS_UNKNOWN) {
+ this.estimatedNumRecords = card;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ // no children, so nothing to compute
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ // because there are no inputs, there are no unclosed branches.
+ this.openBranches = Collections.emptyList();
+ }
+
+ @Override
+ public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
+ if (this.cachedPlans != null) {
+ return this.cachedPlans;
+ }
+
+ SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")",
+ this.gprops, this.lprops);
+
+ if(!replicatedInput) {
+ candidate.updatePropertiesWithUniqueSets(getUniqueFields());
+
+ final Costs costs = new Costs();
+ if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
+ this.estimatedOutputSize >= 0) {
+ estimator.addFileInputCost(this.estimatedOutputSize, costs);
+ }
+ candidate.setCosts(costs);
+ } else {
+ // replicated input
+ final Costs costs = new Costs();
+ InputFormat<?,?> inputFormat =
+ ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
+ if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
+ this.estimatedOutputSize >= 0) {
+ estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs);
+ }
+ candidate.setCosts(costs);
+ }
+
+ // since there is only a single plan for the data-source, return a list with that element only
+ List<PlanNode> plans = new ArrayList<PlanNode>(1);
+ plans.add(candidate);
+
+ this.cachedPlans = plans;
+ return plans;
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new EmptySemanticProperties();
+ }
+
+ @Override
+ public void accept(Visitor<OptimizerNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+
+ private void setDataPropertiesFromSplitProperties(SplitDataProperties splitProps) {
+
+ // set global properties
+ int[] partitionKeys = splitProps.getSplitPartitionKeys();
+ Partitioner<?> partitioner = splitProps.getSplitPartitioner();
+
+ if(partitionKeys != null && partitioner != null) {
+ this.gprops.setCustomPartitioned(new FieldList(partitionKeys), partitioner);
+ }
+ else if(partitionKeys != null) {
+ this.gprops.setAnyPartitioning(new FieldList(partitionKeys));
+ }
+ // set local properties
+ int[] groupingKeys = splitProps.getSplitGroupKeys();
+ Ordering ordering = splitProps.getSplitOrder();
+
+ // more than one split per source tasks possible.
+ // adapt split grouping and sorting
+ if(ordering != null) {
+
+ // sorting falls back to grouping because a source can read multiple,
+ // randomly assigned splits
+ groupingKeys = ordering.getFieldPositions();
+ }
+
+ if(groupingKeys != null && partitionKeys != null) {
+ // check if grouping is also valid across splits, i.e., whether grouping keys are
+ // valid superset of partition keys
+ boolean allFieldsIncluded = true;
+ for(int i : partitionKeys) {
+ boolean fieldIncluded = false;
+ for(int j : groupingKeys) {
+ if(i == j) {
+ fieldIncluded = true;
+ break;
+ }
+ }
+ if(!fieldIncluded) {
+ allFieldsIncluded = false;
+ break;
+ }
+ }
+ if (allFieldsIncluded) {
+ this.lprops = LocalProperties.forGrouping(new FieldList(groupingKeys));
+ } else {
+ this.lprops = new LocalProperties();
+ }
+
+ } else {
+ this.lprops = new LocalProperties();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
new file mode 100644
index 0000000..482951b
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/EstimateProvider.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+/**
+ * Methods for operators / connections that provide estimated about data size and
+ * characteristics.
+ */
+public interface EstimateProvider {
+
+ /**
+ * Gets the estimated output size from this node.
+ *
+ * @return The estimated output size.
+ */
+ long getEstimatedOutputSize();
+
+ /**
+ * Gets the estimated number of records in the output of this node.
+ *
+ * @return The estimated number of records.
+ */
+ long getEstimatedNumRecords();
+
+ /**
+ * Gets the estimated number of bytes per record.
+ *
+ * @return The estimated number of bytes per record.
+ */
+ float getEstimatedAvgWidthPerOutputRecord();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
new file mode 100644
index 0000000..118ddc8
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FilterNode.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.base.FilterOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.FilterDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+/**
+ * The optimizer's internal representation of a <i>FlatMap</i> operator node.
+ */
+public class FilterNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ public FilterNode(FilterOperatorBase<?, ?> operator) {
+ super(operator);
+ this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new FilterDescriptor());
+ }
+
+ @Override
+ public FilterOperatorBase<?, ?> getOperator() {
+ return (FilterOperatorBase<?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Filter";
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ /**
+ * Computes the estimates for the Filter operator. Since it applies a filter on the data we assume a cardinality
+ * decrease. To give the system a hint at data decrease, we use a default magic number to indicate a 0.5 decrease.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = (long) (getPredecessorNode().getEstimatedNumRecords() * 0.5);
+ this.estimatedOutputSize = (long) (getPredecessorNode().getEstimatedOutputSize() * 0.5);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
new file mode 100644
index 0000000..f713d56
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/FlatMapNode.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.FlatMapDescriptor;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+/**
+ * The optimizer's internal representation of a <i>FlatMap</i> operator node.
+ */
+public class FlatMapNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ public FlatMapNode(FlatMapOperatorBase<?, ?, ?> operator) {
+ super(operator);
+
+ this.possibleProperties = Collections.<OperatorDescriptorSingle>singletonList(new FlatMapDescriptor());
+ }
+
+ @Override
+ public FlatMapOperatorBase<?, ?, ?> getOperator() {
+ return (FlatMapOperatorBase<?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "FlatMap";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ /**
+ * Computes the estimates for the FlatMap operator. Since it un-nests, we assume a cardinality
+ * increase. To give the system a hint at data increase, we take a default magic number of a 5 times increase.
+ */
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords() * 5;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
new file mode 100644
index 0000000..564c0d3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupCombineNode.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.operators.AllGroupCombineProperties;
+import org.apache.flink.optimizer.operators.GroupCombineProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The optimizer representation of a <i>GroupCombineNode</i> operation.
+ */
+public class GroupCombineNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ /**
+ * Creates a new optimizer node for the given operator.
+ *
+ * @param operator The reduce operation.
+ */
+ public GroupCombineNode(GroupCombineOperatorBase<?, ?, ?> operator) {
+ super(operator);
+
+ if (this.keys == null) {
+ // case of a key-less reducer. force a parallelism of 1
+ setDegreeOfParallelism(1);
+ }
+
+ this.possibleProperties = initPossibleProperties();
+ }
+
+ private List<OperatorDescriptorSingle> initPossibleProperties() {
+
+ // check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
+ Ordering groupOrder = getOperator().getGroupOrder();
+ if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
+ groupOrder = null;
+ }
+
+ OperatorDescriptorSingle props = (this.keys == null ?
+ new AllGroupCombineProperties() :
+ new GroupCombineProperties(this.keys, groupOrder));
+
+ return Collections.singletonList(props);
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the operator represented by this optimizer node.
+ *
+ * @return The operator represented by this optimizer node.
+ */
+ @Override
+ public GroupCombineOperatorBase<?, ?, ?> getOperator() {
+ return (GroupCombineOperatorBase<?, ?, ?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "GroupCombine";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Estimates
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // no real estimates possible for a reducer.
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
new file mode 100644
index 0000000..77acae5
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/GroupReduceNode.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.operators.AllGroupReduceProperties;
+import org.apache.flink.optimizer.operators.AllGroupWithPartialPreGroupProperties;
+import org.apache.flink.optimizer.operators.GroupReduceProperties;
+import org.apache.flink.optimizer.operators.GroupReduceWithCombineProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * The optimizer representation of a <i>GroupReduce</i> operation.
+ */
+public class GroupReduceNode extends SingleInputNode {
+
+ private final List<OperatorDescriptorSingle> possibleProperties;
+
+ private GroupReduceNode combinerUtilityNode;
+
+ /**
+ * Creates a new optimizer node for the given operator.
+ *
+ * @param operator The reduce operation.
+ */
+ public GroupReduceNode(GroupReduceOperatorBase<?, ?, ?> operator) {
+ super(operator);
+
+ if (this.keys == null) {
+ // case of a key-less reducer. force a parallelism of 1
+ setDegreeOfParallelism(1);
+ }
+
+ this.possibleProperties = initPossibleProperties(operator.getCustomPartitioner());
+ }
+
+ public GroupReduceNode(GroupReduceNode reducerToCopyForCombiner) {
+ super(reducerToCopyForCombiner);
+
+ this.possibleProperties = Collections.emptyList();
+ }
+
+ private List<OperatorDescriptorSingle> initPossibleProperties(Partitioner<?> customPartitioner) {
+ // see if an internal hint dictates the strategy to use
+ final Configuration conf = getOperator().getParameters();
+ final String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
+
+ final boolean useCombiner;
+ if (localStrategy != null) {
+ if (Optimizer.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
+ useCombiner = false;
+ }
+ else if (Optimizer.HINT_LOCAL_STRATEGY_COMBINING_SORT.equals(localStrategy)) {
+ if (!isCombineable()) {
+ Optimizer.LOG.warn("Strategy hint for GroupReduce '" + getOperator().getName() +
+ "' requires combinable reduce, but user function is not marked combinable.");
+ }
+ useCombiner = true;
+ } else {
+ throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
+ }
+ } else {
+ useCombiner = isCombineable();
+ }
+
+ // check if we can work with a grouping (simple reducer), or if we need ordering because of a group order
+ Ordering groupOrder = null;
+ if (getOperator() instanceof GroupReduceOperatorBase) {
+ groupOrder = ((GroupReduceOperatorBase<?, ?, ?>) getOperator()).getGroupOrder();
+ if (groupOrder != null && groupOrder.getNumberOfFields() == 0) {
+ groupOrder = null;
+ }
+ }
+
+ OperatorDescriptorSingle props = useCombiner ?
+ (this.keys == null ? new AllGroupWithPartialPreGroupProperties() : new GroupReduceWithCombineProperties(this.keys, groupOrder, customPartitioner)) :
+ (this.keys == null ? new AllGroupReduceProperties() : new GroupReduceProperties(this.keys, groupOrder, customPartitioner));
+
+ return Collections.singletonList(props);
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the operator represented by this optimizer node.
+ *
+ * @return The operator represented by this optimizer node.
+ */
+ @Override
+ public GroupReduceOperatorBase<?, ?, ?> getOperator() {
+ return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
+ }
+
+ /**
+ * Checks, whether a combiner function has been given for the function encapsulated
+ * by this reduce contract.
+ *
+ * @return True, if a combiner has been given, false otherwise.
+ */
+ public boolean isCombineable() {
+ return getOperator().isCombinable();
+ }
+
+ @Override
+ public String getName() {
+ return "GroupReduce";
+ }
+
+ @Override
+ protected List<OperatorDescriptorSingle> getPossibleProperties() {
+ return this.possibleProperties;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Estimates
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // no real estimates possible for a reducer.
+ }
+
+ public GroupReduceNode getCombinerUtilityNode() {
+ if (this.combinerUtilityNode == null) {
+ this.combinerUtilityNode = new GroupReduceNode(this);
+
+ // we conservatively assume the combiner returns the same data size as it consumes
+ this.combinerUtilityNode.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
+ this.combinerUtilityNode.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
+ }
+ return this.combinerUtilityNode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
new file mode 100644
index 0000000..1fdae51
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/InterestingPropertiesClearer.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.util.Visitor;
+
+final class InterestingPropertiesClearer implements Visitor<OptimizerNode> {
+
+ static final InterestingPropertiesClearer INSTANCE = new InterestingPropertiesClearer();
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ if (visitable.getInterestingProperties() != null) {
+ visitable.clearInterestingProperties();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
new file mode 100644
index 0000000..5d28043
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/IterationNode.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.util.Visitor;
+
+/**
+ *
+ */
+public interface IterationNode {
+
+ void acceptForStepFunction(Visitor<OptimizerNode> visitor);
+
+}
[53/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
[FLINK-441] [optimizer] Rename Pact* and Nephele* classes
Also clean up and improve various comments and method names.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a9150b30
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a9150b30
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a9150b30
Branch: refs/heads/master
Commit: a9150b30f45cf2280130503667e5615b9471ee0a
Parents: 9d222ca
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 11:24:35 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
.../org/apache/flink/client/LocalExecutor.java | 14 +-
.../org/apache/flink/client/program/Client.java | 16 +-
.../flink/client/program/PackagedProgram.java | 8 +-
.../apache/flink/client/program/ClientTest.java | 16 +-
.../flink/optimizer/CompilerException.java | 9 +-
.../optimizer/CompilerPostPassException.java | 8 +-
.../apache/flink/optimizer/DataStatistics.java | 7 +-
.../org/apache/flink/optimizer/Optimizer.java | 1372 +++++++++++++++
.../apache/flink/optimizer/PactCompiler.java | 1372 ---------------
.../dag/AbstractPartialSolutionNode.java | 5 +-
.../flink/optimizer/dag/BinaryUnionNode.java | 8 +-
.../flink/optimizer/dag/BulkIterationNode.java | 26 +-
.../optimizer/dag/BulkPartialSolutionNode.java | 13 +-
.../apache/flink/optimizer/dag/CoGroupNode.java | 6 +-
.../apache/flink/optimizer/dag/CrossNode.java | 16 +-
.../flink/optimizer/dag/DagConnection.java | 290 ++++
.../flink/optimizer/dag/DataSinkNode.java | 32 +-
.../flink/optimizer/dag/DataSourceNode.java | 36 +-
.../flink/optimizer/dag/EstimateProvider.java | 6 +-
.../apache/flink/optimizer/dag/FilterNode.java | 4 +-
.../apache/flink/optimizer/dag/FlatMapNode.java | 4 +-
.../flink/optimizer/dag/GroupCombineNode.java | 6 +-
.../flink/optimizer/dag/GroupReduceNode.java | 22 +-
.../apache/flink/optimizer/dag/JoinNode.java | 20 +-
.../apache/flink/optimizer/dag/MatchNode.java | 20 +-
.../flink/optimizer/dag/OptimizerNode.java | 216 +--
.../flink/optimizer/dag/PactConnection.java | 290 ----
.../flink/optimizer/dag/PartitionNode.java | 6 +-
.../apache/flink/optimizer/dag/ReduceNode.java | 4 +-
.../flink/optimizer/dag/SingleInputNode.java | 48 +-
.../apache/flink/optimizer/dag/SinkJoiner.java | 11 +-
.../flink/optimizer/dag/SolutionSetNode.java | 8 +-
.../flink/optimizer/dag/SortPartitionNode.java | 4 +-
.../flink/optimizer/dag/TwoInputNode.java | 86 +-
.../optimizer/dag/WorksetIterationNode.java | 26 +-
.../apache/flink/optimizer/dag/WorksetNode.java | 8 +-
.../dataproperties/GlobalProperties.java | 6 +-
.../dataproperties/InterestingProperties.java | 19 +-
.../dataproperties/LocalProperties.java | 2 +-
.../dataproperties/PartitioningProperty.java | 23 +-
.../RequestedLocalProperties.java | 18 +-
.../operators/AllGroupCombineProperties.java | 2 +-
.../operators/AllGroupReduceProperties.java | 2 +-
.../AllGroupWithPartialPreGroupProperties.java | 8 +-
.../operators/AllReduceProperties.java | 8 +-
.../operators/CartesianProductDescriptor.java | 2 +-
.../optimizer/operators/CoGroupDescriptor.java | 2 +-
.../CoGroupWithSolutionSetFirstDescriptor.java | 2 +-
.../operators/CollectorMapDescriptor.java | 2 +-
.../optimizer/operators/FilterDescriptor.java | 2 +-
.../optimizer/operators/FlatMapDescriptor.java | 2 +-
.../operators/GroupCombineProperties.java | 4 +-
.../operators/GroupReduceProperties.java | 2 +-
.../GroupReduceWithCombineProperties.java | 8 +-
.../operators/HashJoinBuildFirstProperties.java | 2 +-
.../HashJoinBuildSecondProperties.java | 2 +-
.../optimizer/operators/MapDescriptor.java | 2 +-
.../operators/MapPartitionDescriptor.java | 2 +-
.../operators/PartialGroupProperties.java | 6 +-
.../optimizer/operators/ReduceProperties.java | 8 +-
.../operators/SortMergeJoinDescriptor.java | 2 +-
.../optimizer/plan/BinaryUnionPlanNode.java | 2 +-
.../apache/flink/optimizer/plan/Channel.java | 25 +-
.../flink/optimizer/plan/DualInputPlanNode.java | 4 +-
.../apache/flink/optimizer/plan/PlanNode.java | 29 +-
.../optimizer/plan/SingleInputPlanNode.java | 2 +-
.../plandump/PlanJSONDumpGenerator.java | 24 +-
.../plantranslate/JobGraphGenerator.java | 1578 +++++++++++++++++
.../plantranslate/NepheleJobGraphGenerator.java | 1581 ------------------
.../postpass/GenericFlatTypePostPass.java | 48 +-
.../optimizer/postpass/JavaApiPostPass.java | 18 +-
.../optimizer/postpass/RecordModelPostPass.java | 6 +-
.../optimizer/BranchingPlansCompilerTest.java | 24 +-
.../CachedMatchStrategyCompilerTest.java | 26 +-
.../flink/optimizer/CompilerTestBase.java | 12 +-
.../apache/flink/optimizer/DOPChangeTest.java | 4 +-
.../flink/optimizer/DisjointDataFlowsTest.java | 4 +-
.../optimizer/DistinctCompilationTest.java | 28 +-
.../optimizer/HardPlansCompilationTest.java | 4 +-
.../flink/optimizer/IterationsCompilerTest.java | 14 +-
.../flink/optimizer/NestedIterationsTest.java | 6 +-
.../flink/optimizer/PipelineBreakerTest.java | 8 +-
.../apache/flink/optimizer/ReduceAllTest.java | 4 +-
.../SemanticPropertiesAPIToPlanTest.java | 6 +-
.../UnionBetweenDynamicAndStaticPathTest.java | 6 +-
.../optimizer/UnionPropertyPropagationTest.java | 12 +-
.../flink/optimizer/UnionReplacementTest.java | 4 +-
.../WorksetIterationCornerCasesTest.java | 4 +-
.../WorksetIterationsRecordApiCompilerTest.java | 8 +-
...naryCustomPartitioningCompatibilityTest.java | 6 +-
.../custompartition/CustomPartitioningTest.java | 28 +-
.../DataExchangeModeClosedBranchingTest.java | 2 +-
.../DataExchangeModeOpenBranchingTest.java | 2 +-
.../dataexchange/PipelineBreakingTest.java | 10 +-
.../java/GroupReduceCompilationTest.java | 50 +-
.../optimizer/java/IterationCompilerTest.java | 10 +-
.../optimizer/java/ReduceCompilationTest.java | 34 +-
.../WorksetIterationsJavaApiCompilerTest.java | 8 +-
.../CoGroupOnConflictingPartitioningsTest.java | 6 +-
.../JoinOnConflictingPartitioningsTest.java | 6 +-
.../flink/spargel/java/SpargelCompilerTest.java | 16 +-
.../test/compiler/util/CompilerTestBase.java | 14 +-
.../flink/test/util/RecordAPITestBase.java | 8 +-
.../apache/flink/test/util/TestEnvironment.java | 8 +-
.../test/cancelling/CancellingTestBase.java | 8 +-
.../ConnectedComponentsCoGroupTest.java | 4 +-
.../iterations/ConnectedComponentsTest.java | 6 +-
.../iterations/IterativeKMeansTest.java | 6 +-
...ultipleJoinsWithSolutionSetCompilerTest.java | 4 +-
.../iterations/PageRankCompilerTest.java | 4 +-
.../compiler/plandump/PreviewPlanDumpTest.java | 4 +-
.../exampleJavaPrograms/WordCountITCase.java | 5 -
.../test/failingPrograms/TaskFailureITCase.java | 12 +-
.../hadoop/mapred/WordCountMapredITCase.java | 7 -
.../mapreduce/WordCountMapreduceITCase.java | 6 -
.../javaApiOperators/GroupReduceITCase.java | 4 +-
.../flink/test/operators/CoGroupITCase.java | 10 +-
.../flink/test/operators/CrossITCase.java | 30 +-
.../apache/flink/test/operators/JoinITCase.java | 28 +-
.../flink/test/operators/ReduceITCase.java | 16 +-
.../flink/test/operators/UnionSinkITCase.java | 8 +-
.../api/scala/operators/GroupReduceITCase.scala | 4 +-
.../translation/CustomPartitioningTest.scala | 28 +-
123 files changed, 4042 insertions(+), 4032 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
index a2aed8f..6df0f79 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
@@ -33,11 +33,11 @@ import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
/**
@@ -158,10 +158,10 @@ public class LocalExecutor extends PlanExecutor {
}
try {
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
JobGraph jobGraph = jgg.compileJobGraph(op);
ActorRef jobClient = flink.getJobClient();
@@ -186,7 +186,7 @@ public class LocalExecutor extends PlanExecutor {
* @throws Exception
*/
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();
@@ -242,7 +242,7 @@ public class LocalExecutor extends PlanExecutor {
LocalExecutor exec = new LocalExecutor();
try {
exec.start();
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();
@@ -260,7 +260,7 @@ public class LocalExecutor extends PlanExecutor {
*/
public static String getPlanAsJSON(Plan plan) {
PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();
- List<DataSinkNode> sinks = PactCompiler.createPreOptimizedPlan(plan);
+ List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
return gen.getPactPlanAsJSON(sinks);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
index 84e6637..4e593c8 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
@@ -32,13 +32,13 @@ import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.ExecutionEnvironmentFactory;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
import org.apache.flink.optimizer.plan.FlinkPlan;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.StreamingPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
@@ -66,7 +66,7 @@ public class Client {
private final Configuration configuration; // the configuration describing the job manager address
- private final PactCompiler compiler; // the compiler to compile the jobs
+ private final Optimizer compiler; // the compiler to compile the jobs
private boolean printStatusDuringExecution = false;
@@ -88,7 +88,7 @@ public class Client {
configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerAddress.getAddress().getHostAddress());
configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerAddress.getPort());
- this.compiler = new PactCompiler(new DataStatistics(), new DefaultCostEstimator());
+ this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator());
}
/**
@@ -112,7 +112,7 @@ public class Client {
throw new CompilerException("Cannot find port to job manager's RPC service in the global configuration.");
}
- this.compiler = new PactCompiler(new DataStatistics(), new DefaultCostEstimator());
+ this.compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator());
}
public void setPrintStatusDuringExecution(boolean print) {
@@ -219,7 +219,7 @@ public class Client {
if (optPlan instanceof StreamingPlan) {
job = ((StreamingPlan) optPlan).getJobGraph();
} else {
- NepheleJobGraphGenerator gen = new NepheleJobGraphGenerator();
+ JobGraphGenerator gen = new JobGraphGenerator();
job = gen.compileJobGraph((OptimizedPlan) optPlan);
}
@@ -355,12 +355,12 @@ public class Client {
public static final class OptimizerPlanEnvironment extends ExecutionEnvironment {
- private final PactCompiler compiler;
+ private final Optimizer compiler;
private FlinkPlan optimizerPlan;
- private OptimizerPlanEnvironment(PactCompiler compiler) {
+ private OptimizerPlanEnvironment(Optimizer compiler) {
this.compiler = compiler;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgram.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgram.java b/flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgram.java
index d79d404..10096da 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgram.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgram.java
@@ -46,7 +46,7 @@ import org.apache.flink.api.common.Program;
import org.apache.flink.api.common.ProgramDescription;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.ExecutionEnvironmentFactory;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.util.InstantiationUtil;
@@ -255,7 +255,7 @@ public class PackagedProgram {
List<DataSinkNode> previewPlan;
if (isUsingProgramEntryPoint()) {
- previewPlan = PactCompiler.createPreOptimizedPlan(getPlan());
+ previewPlan = Optimizer.createPreOptimizedPlan(getPlan());
}
else if (isUsingInteractiveMode()) {
// temporary hack to support the web client
@@ -698,7 +698,7 @@ public class PackagedProgram {
@Override
public JobExecutionResult execute(String jobName) throws Exception {
this.plan = createProgramPlan(jobName);
- this.previewPlan = PactCompiler.createPreOptimizedPlan((Plan) plan);
+ this.previewPlan = Optimizer.createPreOptimizedPlan((Plan) plan);
// do not go on with anything now!
throw new Client.ProgramAbortException();
@@ -707,7 +707,7 @@ public class PackagedProgram {
@Override
public String getExecutionPlan() throws Exception {
Plan plan = createProgramPlan("unused");
- this.previewPlan = PactCompiler.createPreOptimizedPlan(plan);
+ this.previewPlan = Optimizer.createPreOptimizedPlan(plan);
// do not go on with anything now!
throw new Client.ProgramAbortException();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
index bac5c04..9278c7a 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/ClientTest.java
@@ -27,10 +27,10 @@ import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.akka.AkkaUtils;
@@ -69,8 +69,8 @@ import static org.powermock.api.mockito.PowerMockito.whenNew;
public class ClientTest {
private PackagedProgram program;
- private PactCompiler compilerMock;
- private NepheleJobGraphGenerator generatorMock;
+ private Optimizer compilerMock;
+ private JobGraphGenerator generatorMock;
private Configuration config;
@@ -89,8 +89,8 @@ public class ClientTest {
config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, ConfigConstants.DEFAULT_AKKA_ASK_TIMEOUT);
program = mock(PackagedProgram.class);
- compilerMock = mock(PactCompiler.class);
- generatorMock = mock(NepheleJobGraphGenerator.class);
+ compilerMock = mock(Optimizer.class);
+ generatorMock = mock(JobGraphGenerator.class);
JobWithJars planWithJarsMock = mock(JobWithJars.class);
Plan planMock = mock(Plan.class);
@@ -101,10 +101,10 @@ public class ClientTest {
when(program.getPlanWithJars()).thenReturn(planWithJarsMock);
when(planWithJarsMock.getPlan()).thenReturn(planMock);
- whenNew(PactCompiler.class).withArguments(any(DataStatistics.class), any(CostEstimator.class)).thenReturn(this.compilerMock);
+ whenNew(Optimizer.class).withArguments(any(DataStatistics.class), any(CostEstimator.class)).thenReturn(this.compilerMock);
when(compilerMock.compile(planMock)).thenReturn(optimizedPlanMock);
- whenNew(NepheleJobGraphGenerator.class).withNoArguments().thenReturn(generatorMock);
+ whenNew(JobGraphGenerator.class).withNoArguments().thenReturn(generatorMock);
when(generatorMock.compileJobGraph(optimizedPlanMock)).thenReturn(jobGraph);
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
index 999c687..2f99ddb 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
@@ -16,24 +16,19 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer;
/**
- * An exception that is thrown by the pact compiler when encountering an illegal condition.
+ * An exception that is thrown by the Optimizer when encountering an illegal condition.
*/
public class CompilerException extends RuntimeException {
- /**
- * Serial version UID for serialization interoperability.
- */
private static final long serialVersionUID = 3810067304570563755L;
/**
* Creates a compiler exception with no message and no cause.
*/
- public CompilerException() {
- }
+ public CompilerException() {}
/**
* Creates a compiler exception with the given message and no cause.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
index 1c58217..78e47a0 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
@@ -16,19 +16,15 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer;
/**
- * An exception that is thrown by the pact compiler when encountering
+ * An exception that is thrown by the Optimizer when encountering
* a problem during the optimizer post pass. This is a dedicated exception
* because it is thrown by user-specified optimizer extensions.
*/
public class CompilerPostPassException extends CompilerException {
-
- /**
- * Serial version UID for serialization interoperability.
- */
+
private static final long serialVersionUID = -322650826288034623L;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
index 1d43b76..cf6f4ec 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer;
import java.util.HashMap;
@@ -60,11 +59,11 @@ public class DataStatistics {
* Caches the given statistics. They are later retrievable under the given identifier.
*
* @param statistics The statistics to cache.
- * @param identifyer The identifier which may be later used to retrieve the statistics.
+ * @param identifier The identifier which may be later used to retrieve the statistics.
*/
- public void cacheBaseStatistics(BaseStatistics statistics, String identifyer) {
+ public void cacheBaseStatistics(BaseStatistics statistics, String identifier) {
synchronized (this.baseStatisticsCache) {
- this.baseStatisticsCache.put(identifyer, statistics);
+ this.baseStatisticsCache.put(identifier, statistics);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
new file mode 100644
index 0000000..e76501c
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
@@ -0,0 +1,1372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
+import org.apache.flink.optimizer.dag.GroupCombineNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
+import org.apache.flink.optimizer.dag.SortPartitionNode;
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.GenericDataSinkBase;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Union;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
+import org.apache.flink.api.common.operators.base.CrossOperatorBase;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.api.common.operators.base.FilterOperatorBase;
+import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.BulkIterationBase.PartialSolutionPlaceHolder;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.costs.DefaultCostEstimator;
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dag.BulkIterationNode;
+import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
+import org.apache.flink.optimizer.dag.CoGroupNode;
+import org.apache.flink.optimizer.dag.CollectorMapNode;
+import org.apache.flink.optimizer.dag.CrossNode;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dag.FilterNode;
+import org.apache.flink.optimizer.dag.FlatMapNode;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.MapNode;
+import org.apache.flink.optimizer.dag.MapPartitionNode;
+import org.apache.flink.optimizer.dag.JoinNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.PartitionNode;
+import org.apache.flink.optimizer.dag.ReduceNode;
+import org.apache.flink.optimizer.dag.SinkJoiner;
+import org.apache.flink.optimizer.dag.SolutionSetNode;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.optimizer.dag.WorksetNode;
+import org.apache.flink.optimizer.deadlockdetect.DeadlockPreventer;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.optimizer.postpass.OptimizerPostPass;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The optimizer that takes the user specified program plan and creates an optimized plan that contains
+ * exact descriptions about how the physical execution will take place. It first translates the user
+ * program into an internal optimizer representation and then chooses between different alternatives
+ * for shipping strategies and local strategies.
+ * <p>
+ * The basic principle is taken from optimizer works in systems such as Volcano/Cascades and Selinger/System-R/DB2. The
+ * optimizer walks from the sinks down, generating interesting properties, and ascends from the sources generating
+ * alternative plans, pruning against the interesting properties.
+ * <p>
+ * The optimizer also assigns the memory to the individual tasks. This is currently done in a very simple fashion: All
+ * sub-tasks that need memory (e.g. reduce or join) are given an equal share of memory.
+ */
+public class Optimizer {
+
+ // ------------------------------------------------------------------------
+ // Constants
+ // ------------------------------------------------------------------------
+
+ /**
+ * Compiler hint key for the input channel's shipping strategy. This String is a key to the operator's stub
+ * parameters. The corresponding value tells the compiler which shipping strategy to use for the input channel.
+ * If the operator has two input channels, the shipping strategy is applied to both input channels.
+ */
+ public static final String HINT_SHIP_STRATEGY = "INPUT_SHIP_STRATEGY";
+
+ /**
+ * Compiler hint key for the <b>first</b> input channel's shipping strategy. This String is a key to
+ * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
+ * to use for the <b>first</b> input channel. Only applicable to operators with two inputs.
+ */
+ public static final String HINT_SHIP_STRATEGY_FIRST_INPUT = "INPUT_LEFT_SHIP_STRATEGY";
+
+ /**
+ * Compiler hint key for the <b>second</b> input channel's shipping strategy. This String is a key to
+ * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
+ * to use for the <b>second</b> input channel. Only applicable to operators with two inputs.
+ */
+ public static final String HINT_SHIP_STRATEGY_SECOND_INPUT = "INPUT_RIGHT_SHIP_STRATEGY";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a <b>Forward</b> strategy on the
+ * input channel, i.e. no redistribution of any kind.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_FORWARD = "SHIP_FORWARD";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a random repartition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION= "SHIP_REPARTITION";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a hash-partition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION_HASH = "SHIP_REPARTITION_HASH";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a range-partition strategy.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_REPARTITION_RANGE = "SHIP_REPARTITION_RANGE";
+
+ /**
+ * Value for the shipping strategy compiler hint that enforces a <b>broadcast</b> strategy on the
+ * input channel.
+ *
+ * @see #HINT_SHIP_STRATEGY
+ * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
+ * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
+ */
+ public static final String HINT_SHIP_STRATEGY_BROADCAST = "SHIP_BROADCAST";
+
+ /**
+ * Compiler hint key for the operator's local strategy. This String is a key to the operator's stub
+ * parameters. The corresponding value tells the compiler which local strategy to use to process the
+ * data inside one partition.
+ * <p>
+ * This hint is ignored by operators that do not have a local strategy (such as <i>Map</i>), or by operators that
+ * have no choice in their local strategy (such as <i>Cross</i>).
+ */
+ public static final String HINT_LOCAL_STRATEGY = "LOCAL_STRATEGY";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
+ * For example, a <i>Reduce</i> operator will sort the data to group it.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT = "LOCAL_STRATEGY_SORT";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
+ * During sorting a combine method is repeatedly applied to reduce the data volume.
+ * For example, a <i>Reduce</i> operator will sort the data to group it.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_COMBINING_SORT = "LOCAL_STRATEGY_COMBINING_SORT";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy on both
+ * inputs with subsequent merging of inputs.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE = "LOCAL_STRATEGY_SORT_BOTH_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
+ * The first input is sorted, the second input is assumed to be sorted. After sorting both inputs are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE = "LOCAL_STRATEGY_SORT_FIRST_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
+ * The second input is sorted, the first input is assumed to be sorted. After sorting both inputs are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE = "LOCAL_STRATEGY_SORT_SECOND_MERGE";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>merge based</b> local strategy.
+ * Both inputs are assumed to be sorted and are merged.
+ * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a merge strategy to find pairs
+ * of matching keys.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_MERGE = "LOCAL_STRATEGY_MERGE";
+
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
+ * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
+ * matching keys. The <b>first</b> input will be used to build the hash table, the second input will be
+ * used to probe the table.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST = "LOCAL_STRATEGY_HASH_BUILD_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
+ * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
+ * matching keys. The <b>second</b> input will be used to build the hash table, the first input will be
+ * used to probe the table.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND = "LOCAL_STRATEGY_HASH_BUILD_SECOND";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
+ * Hence, the data of the first input will be is streamed though, while the data of the second input is stored on
+ * disk
+ * and repeatedly read.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
+ * Hence, the data of the second input will be is streamed though, while the data of the first input is stored on
+ * disk
+ * and repeatedly read.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
+ * Further more, the first input, being the outer side, will be processed in blocks, and for each block, the second
+ * input,
+ * being the inner side, will read repeatedly from disk.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST";
+
+ /**
+ * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
+ * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
+ * Further more, the second input, being the outer side, will be processed in blocks, and for each block, the first
+ * input,
+ * being the inner side, will read repeatedly from disk.
+ *
+ * @see #HINT_LOCAL_STRATEGY
+ */
+ public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND";
+
+ /**
+ * The log handle that is used by the compiler to log messages.
+ */
+ public static final Logger LOG = LoggerFactory.getLogger(Optimizer.class);
+
+ // ------------------------------------------------------------------------
+ // Members
+ // ------------------------------------------------------------------------
+
+ /**
+ * The statistics object used to obtain statistics, such as input sizes,
+ * for the cost estimation process.
+ */
+ private final DataStatistics statistics;
+
+ /**
+ * The cost estimator used by the compiler.
+ */
+ private final CostEstimator costEstimator;
+
+ /**
+ * The default degree of parallelism for jobs compiled by this compiler.
+ */
+ private int defaultDegreeOfParallelism;
+
+
+ // ------------------------------------------------------------------------
+ // Constructor & Setup
+ // ------------------------------------------------------------------------
+
+ /**
+ * Creates a new optimizer instance. The optimizer has no access to statistics about the
+ * inputs and can hence not determine any properties. It will perform all optimization with
+ * unknown sizes and hence use only the heuristic cost functions, which result in the selection
+ * of the most robust execution strategies.
+ */
+ public Optimizer() {
+ this(null, new DefaultCostEstimator());
+ }
+
+ /**
+ * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
+ * Given those statistics, the optimizer can make better choices for the execution strategies.
+ *
+ * @param stats
+ * The statistics to be used to determine the input properties.
+ */
+ public Optimizer(DataStatistics stats) {
+ this(stats, new DefaultCostEstimator());
+ }
+
+ /**
+ * Creates a new optimizer instance. The optimizer has no access to statistics about the
+ * inputs and can hence not determine any properties. It will perform all optimization with
+ * unknown sizes and hence use only the heuristic cost functions, which result in the selection
+ * of the most robust execution strategies.
+ *
+ * The optimizer uses the given cost estimator to compute the costs of the individual operations.
+ *
+ * @param estimator The cost estimator to use to cost the individual operations.
+ */
+ public Optimizer(CostEstimator estimator) {
+ this(null, estimator);
+ }
+
+ /**
+ * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
+ * Given those statistics, the optimizer can make better choices for the execution strategies.
+ *
+ * The optimizer uses the given cost estimator to compute the costs of the individual operations.
+ *
+ * @param stats
+ * The statistics to be used to determine the input properties.
+ * @param estimator
+ * The <tt>CostEstimator</tt> to use to cost the individual operations.
+ */
+ public Optimizer(DataStatistics stats, CostEstimator estimator) {
+ this.statistics = stats;
+ this.costEstimator = estimator;
+
+ // determine the default parallelism
+ this.defaultDegreeOfParallelism = GlobalConfiguration.getInteger(
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
+ ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE);
+
+ if (defaultDegreeOfParallelism < 1) {
+ LOG.warn("Config value " + defaultDegreeOfParallelism + " for option "
+ + ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE + " is invalid. Ignoring and using a value of 1.");
+ this.defaultDegreeOfParallelism = 1;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Getters / Setters
+ // ------------------------------------------------------------------------
+
+ public int getDefaultDegreeOfParallelism() {
+ return defaultDegreeOfParallelism;
+ }
+
+ public void setDefaultDegreeOfParallelism(int defaultDegreeOfParallelism) {
+ if (defaultDegreeOfParallelism > 0) {
+ this.defaultDegreeOfParallelism = defaultDegreeOfParallelism;
+ } else {
+ throw new IllegalArgumentException("Default parallelism cannot be zero or negative.");
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Compilation
+ // ------------------------------------------------------------------------
+
+ /**
+ * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned
+ * and all channels have a shipping strategy assigned.
+ *
+ * For more details on the optimization phase, see the comments for
+ * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}.
+ *
+ * @param program The program to be translated.
+ * @return The optimized plan.
+ *
+ * @throws CompilerException
+ * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
+ * situation during the compilation process.
+ */
+ public OptimizedPlan compile(Plan program) throws CompilerException {
+ final OptimizerPostPass postPasser = getPostPassFromPlan(program);
+ return compile(program, postPasser);
+ }
+
+ /**
+ * Translates the given program to an OptimizedPlan. The optimized plan describes for each operator
+ * which strategy to use (such as hash join versus sort-merge join), what data exchange method to use
+ * (local pipe forward, shuffle, broadcast), what exchange mode to use (pipelined, batch),
+ * where to cache intermediate results, etc,
+ *
+ * The optimization happens in multiple phases:
+ * <ol>
+ * <li>Create optimizer dag implementation of the program.
+ *
+ * <tt>OptimizerNode</tt> representations of the PACTs, assign parallelism and compute size estimates.</li>
+ * <li>Compute interesting properties and auxiliary structures.</li>
+ * <li>Enumerate plan alternatives. This cannot be done in the same step as the interesting property computation (as
+ * opposed to the Database approaches), because we support plans that are not trees.</li>
+ * </ol>
+ *
+ * @param program The program to be translated.
+ * @param postPasser The function to be used for post passing the optimizer's plan and setting the
+ * data type specific serialization routines.
+ * @return The optimized plan.
+ *
+ * @throws CompilerException
+ * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
+ * situation during the compilation process.
+ */
+ private OptimizedPlan compile(Plan program, OptimizerPostPass postPasser) throws CompilerException {
+ if (program == null || postPasser == null) {
+ throw new NullPointerException();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Beginning compilation of program '" + program.getJobName() + '\'');
+ }
+
+ final ExecutionMode defaultDataExchangeMode = program.getExecutionConfig().getExecutionMode();
+
+ final int defaultParallelism = program.getDefaultParallelism() > 0 ?
+ program.getDefaultParallelism() : this.defaultDegreeOfParallelism;
+
+ // log the default settings
+ LOG.debug("Using a default parallelism of {}", defaultParallelism);
+ LOG.debug("Using default data exchange mode {}", defaultDataExchangeMode);
+
+ // the first step in the compilation is to create the optimizer plan representation
+ // this step does the following:
+ // 1) It creates an optimizer plan node for each operator
+ // 2) It connects them via channels
+ // 3) It looks for hints about local strategies and channel types and
+ // sets the types and strategies accordingly
+ // 4) It makes estimates about the data volume of the data sources and
+ // propagates those estimates through the plan
+
+ GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode);
+ program.accept(graphCreator);
+
+ // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
+ // each until we have only a single root node. This allows to transparently deal with the nodes with
+ // multiple outputs
+ OptimizerNode rootNode;
+ if (graphCreator.sinks.size() == 1) {
+ rootNode = graphCreator.sinks.get(0);
+ } else if (graphCreator.sinks.size() > 1) {
+ Iterator<DataSinkNode> iter = graphCreator.sinks.iterator();
+ rootNode = iter.next();
+
+ while (iter.hasNext()) {
+ rootNode = new SinkJoiner(rootNode, iter.next());
+ }
+ } else {
+ throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
+ }
+
+ // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
+ // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
+ rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
+
+ // We are dealing with operator DAGs, rather than operator trees.
+ // That requires us to deviate at some points from the classical DB optimizer algorithms.
+ // This step build some auxiliary structures to help track branches and joins in the DAG
+ BranchesVisitor branchingVisitor = new BranchesVisitor();
+ rootNode.accept(branchingVisitor);
+
+ // Propagate the interesting properties top-down through the graph
+ InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
+ rootNode.accept(propsVisitor);
+
+ // perform a sanity check: the root may not have any unclosed branches
+ if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
+ throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
+ "track the re-joining of branches correctly.");
+ }
+
+ // the final step is now to generate the actual plan alternatives
+ List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);
+
+ if (bestPlan.size() != 1) {
+ throw new CompilerException("Error in compiler: more than one best plan was created!");
+ }
+
+ // check if the best plan's root is a data sink (single sink plan)
+ // if so, directly take it. if it is a sink joiner node, get its contained sinks
+ PlanNode bestPlanRoot = bestPlan.get(0);
+ List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);
+
+ if (bestPlanRoot instanceof SinkPlanNode) {
+ bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
+ } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {
+ ((SinkJoinerPlanNode) bestPlanRoot).getDataSinks(bestPlanSinks);
+ }
+
+ DeadlockPreventer dp = new DeadlockPreventer();
+ dp.resolveDeadlocks(bestPlanSinks);
+
+ // finalize the plan
+ OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program);
+
+ plan.accept(new BinaryUnionReplacer());
+
+ // post pass the plan. this is the phase where the serialization and comparator code is set
+ postPasser.postPass(plan);
+
+ return plan;
+ }
+
+ /**
+ * This function performs only the first step to the compilation process - the creation of the optimizer
+ * representation of the plan. No estimations or enumerations of alternatives are done here.
+ *
+ * @param program The plan to generate the optimizer representation for.
+ * @return The optimizer representation of the plan, as a collection of all data sinks
+ * from the plan can be traversed.
+ */
+ public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
+ GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
+ program.accept(graphCreator);
+ return graphCreator.sinks;
+ }
+
+ // ------------------------------------------------------------------------
+ // Visitors for Compilation Traversals
+ // ------------------------------------------------------------------------
+
+ /**
+ * This utility class performs the translation from the user specified program to the optimizer plan.
+ * It works as a visitor that walks the user's job in a depth-first fashion. During the descend, it creates
+ * an optimizer node for each operator, respectively data source or -sink. During the ascend, it connects
+ * the nodes to the full graph.
+ * <p>
+ * This translator relies on the <code>setInputs</code> method in the nodes. As that method implements the size
+ * estimation and the awareness for optimizer hints, the sizes will be properly estimated and the translated plan
+ * already respects all optimizer hints.
+ */
+ public static final class GraphCreatingVisitor implements Visitor<Operator<?>> {
+
+ private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
+ // corresponding optimizer nodes
+
+ private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final int defaultParallelism; // the default degree of parallelism
+
+ private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
+
+ private final ExecutionMode defaultDataExchangeMode;
+
+ private final boolean forceDOP;
+
+
+ public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
+ this(null, false, defaultParallelism, defaultDataExchangeMode, null);
+ }
+
+ private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
+ ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
+ if (closure == null){
+ con2node = new HashMap<Operator<?>, OptimizerNode>();
+ } else {
+ con2node = closure;
+ }
+
+ this.sinks = new ArrayList<DataSinkNode>(2);
+ this.defaultParallelism = defaultParallelism;
+ this.parent = parent;
+ this.defaultDataExchangeMode = dataExchangeMode;
+ this.forceDOP = forceDOP;
+ }
+
+ public List<DataSinkNode> getSinks() {
+ return sinks;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public boolean preVisit(Operator<?> c) {
+ // check if we have been here before
+ if (this.con2node.containsKey(c)) {
+ return false;
+ }
+
+ final OptimizerNode n;
+
+ // create a node for the operator (or sink or source) if we have not been here before
+ if (c instanceof GenericDataSinkBase) {
+ DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
+ this.sinks.add(dsn);
+ n = dsn;
+ }
+ else if (c instanceof GenericDataSourceBase) {
+ n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
+ }
+ else if (c instanceof MapOperatorBase) {
+ n = new MapNode((MapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof MapPartitionOperatorBase) {
+ n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
+ n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FlatMapOperatorBase) {
+ n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FilterOperatorBase) {
+ n = new FilterNode((FilterOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof ReduceOperatorBase) {
+ n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof GroupReduceOperatorBase) {
+ n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof GroupCombineOperatorBase) {
+ n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof JoinOperatorBase) {
+ n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CoGroupOperatorBase) {
+ n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CrossOperatorBase) {
+ n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof BulkIterationBase) {
+ n = new BulkIterationNode((BulkIterationBase<?>) c);
+ }
+ else if (c instanceof DeltaIterationBase) {
+ n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
+ }
+ else if (c instanceof Union){
+ n = new BinaryUnionNode((Union<?>) c);
+ }
+ else if (c instanceof PartitionOperatorBase) {
+ n = new PartitionNode((PartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof SortPartitionOperatorBase) {
+ n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof PartialSolutionPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
+ final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
+ final BulkIterationNode containingIterationNode =
+ (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof WorksetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ WorksetNode p = new WorksetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof SolutionSetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else {
+ throw new IllegalArgumentException("Unknown operator type: " + c);
+ }
+
+ this.con2node.put(c, n);
+
+ // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
+ // key-less reducer (all-reduce)
+ if (n.getParallelism() < 1) {
+ // set the degree of parallelism
+ int par = c.getDegreeOfParallelism();
+ if (par > 0) {
+ if (this.forceDOP && par != this.defaultParallelism) {
+ par = this.defaultParallelism;
+ LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
+ "currently fixed to the parallelism of the surrounding operator (the iteration).");
+ }
+ } else {
+ par = this.defaultParallelism;
+ }
+ n.setDegreeOfParallelism(par);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void postVisit(Operator<?> c) {
+
+ OptimizerNode n = this.con2node.get(c);
+
+ // first connect to the predecessors
+ n.setInput(this.con2node, this.defaultDataExchangeMode);
+ n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
+
+ // if the node represents a bulk iteration, we recursively translate the data flow now
+ if (n instanceof BulkIterationNode) {
+ final BulkIterationNode iterNode = (BulkIterationNode) n;
+ final BulkIterationBase<?> iter = iterNode.getIterationContract();
+
+ // pass a copy of the no iterative part into the iteration translation,
+ // in case the iteration references its closure
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
+ iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ BulkPartialSolutionNode partialSolution;
+
+ iter.getNextPartialSolution().accept(recursiveCreator);
+
+ partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
+ OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
+ if (partialSolution == null) {
+ throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
+ }
+
+
+ OptimizerNode terminationCriterion = null;
+
+ if (iter.getTerminationCriterion() != null) {
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+
+ // no intermediate node yet, traverse from the termination criterion to build the missing parts
+ if (terminationCriterion == null) {
+ iter.getTerminationCriterion().accept(recursiveCreator);
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+ }
+ }
+
+ iterNode.setPartialSolution(partialSolution);
+ iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(identifier);
+ }
+ else if (n instanceof WorksetIterationNode) {
+ final WorksetIterationNode iterNode = (WorksetIterationNode) n;
+ final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
+
+ // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
+ // One check is for free during the translation, we do the other check here as a pre-condition
+ {
+ StepFunctionValidator wsf = new StepFunctionValidator();
+ iter.getNextWorkset().accept(wsf);
+ if (!wsf.foundWorkset) {
+ throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
+ "This is a prerequisite in delta iterations.");
+ }
+ }
+
+ // calculate the closure of the anonymous function
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
+ this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ // descend from the solution set delta. check that it depends on both the workset
+ // and the solution set. If it does depend on both, this descend should create both nodes
+ iter.getSolutionSetDelta().accept(recursiveCreator);
+
+ final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
+
+ if (worksetNode == null) {
+ throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
+ "This is a prerequisite in delta iterations.");
+ }
+
+ iter.getNextWorkset().accept(recursiveCreator);
+
+ SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
+
+ if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
+ solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
+ }
+ else {
+ for (DagConnection conn : solutionSetNode.getOutgoingConnections()) {
+ OptimizerNode successor = conn.getTarget();
+
+ if (successor.getClass() == JoinNode.class) {
+ // find out which input to the match the solution set is
+ JoinNode mn = (JoinNode) successor;
+ if (mn.getFirstPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(0);
+ } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else if (successor.getClass() == CoGroupNode.class) {
+ CoGroupNode cg = (CoGroupNode) successor;
+ if (cg.getFirstPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(0);
+ } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else {
+ throw new InvalidProgramException(
+ "Error: The only operations allowed on the solution set are Join and CoGroup.");
+ }
+ }
+ }
+
+ final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
+ final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
+
+ // set the step function nodes to the iteration node
+ iterNode.setPartialSolution(solutionSetNode, worksetNode);
+ iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(pathIdentifier);
+ }
+ }
+ }
+
+ private static final class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
+
+ private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
+
+ private final int costWeight;
+
+ private StaticDynamicPathIdentifier(int costWeight) {
+ this.costWeight = costWeight;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return this.seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ visitable.identifyDynamicPath(this.costWeight);
+
+ // check that there is no nested iteration on the dynamic path
+ if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
+ throw new CompilerException("Nested iterations are currently not supported.");
+ }
+ }
+ }
+
+ /**
+ * Simple visitor that sets the minimal guaranteed memory per task based on the amount of available memory,
+ * the number of memory consumers, and on the task's degree of parallelism.
+ */
+ public static final class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
+
+ private final DataStatistics statistics;
+
+ private int id = 1;
+
+ public IdAndEstimatesVisitor(DataStatistics statistics) {
+ this.statistics = statistics;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return visitable.getId() == -1;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ // the node ids
+ visitable.initId(this.id++);
+
+ // connections need to figure out their maximum path depths
+ for (DagConnection conn : visitable.getIncomingConnections()) {
+ conn.initMaxDepth();
+ }
+ for (DagConnection conn : visitable.getBroadcastConnections()) {
+ conn.initMaxDepth();
+ }
+
+ // the estimates
+ visitable.computeOutputEstimates(this.statistics);
+
+ // if required, recurse into the step function
+ if (visitable instanceof IterationNode) {
+ ((IterationNode) visitable).acceptForStepFunction(this);
+ }
+ }
+ }
+
+ /**
+ * Visitor that computes the interesting properties for each node in the plan. On its recursive
+ * depth-first descend, it propagates all interesting properties top-down.
+ */
+ public static final class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
+
+ private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
+
+ /**
+ * Creates a new visitor that computes the interesting properties for all nodes in the plan.
+ * It uses the given cost estimator used to compute the maximal costs for an interesting property.
+ *
+ * @param estimator
+ * The cost estimator to estimate the maximal costs for interesting properties.
+ */
+ public InterestingPropertyVisitor(CostEstimator estimator) {
+ this.estimator = estimator;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ // The interesting properties must be computed on the descend. In case a node has multiple outputs,
+ // that computation must happen during the last descend.
+
+ if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
+ node.computeUnionOfInterestingPropertiesFromSuccessors();
+ node.computeInterestingPropertiesForInputs(this.estimator);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {}
+ }
+
+ /**
+ * On its re-ascend (post visit) this visitor, computes auxiliary maps that are needed to support plans
+ * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
+ * output into more than one other node).
+ */
+ public static final class BranchesVisitor implements Visitor<OptimizerNode> {
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ return node.getOpenBranches() == null;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode node) {
+ if (node instanceof IterationNode) {
+ ((IterationNode) node).acceptForStepFunction(this);
+ }
+
+ node.computeUnclosedBranchStack();
+ }
+ }
+
+ /**
+ * Finalization of the plan:
+ * - The graph of nodes is double-linked (links from child to parent are inserted)
+ * - If unions join static and dynamic paths, the cache is marked as a memory consumer
+ * - Relative memory fractions are assigned to all nodes.
+ * - All nodes are collected into a set.
+ */
+ private static final class PlanFinalizer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
+
+ private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
+
+ private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final Deque<IterationPlanNode> stackOfIterationNodes;
+
+ private int memoryConsumerWeights; // a counter of all memory consumers
+
+ /**
+ * Creates a new plan finalizer.
+ */
+ private PlanFinalizer() {
+ this.allNodes = new HashSet<PlanNode>();
+ this.sources = new ArrayList<SourcePlanNode>();
+ this.sinks = new ArrayList<SinkPlanNode>();
+ this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
+ }
+
+ private OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
+ this.memoryConsumerWeights = 0;
+
+ // traverse the graph
+ for (SinkPlanNode node : sinks) {
+ node.accept(this);
+ }
+
+ // assign the memory to each node
+ if (this.memoryConsumerWeights > 0) {
+ for (PlanNode node : this.allNodes) {
+ // assign memory to the driver strategy of the node
+ final int consumerWeight = node.getMemoryConsumerWeight();
+ if (consumerWeight > 0) {
+ final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
+ node.setRelativeMemoryPerSubtask(relativeMem);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
+ node.getProgramOperator().getName() + ".");
+ }
+ }
+
+ // assign memory to the local and global strategies of the channels
+ for (Channel c : node.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ final double relativeMem = 1.0 / this.memoryConsumerWeights;
+ c.setRelativeMemoryLocalStrategy(relativeMem);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
+ "instance of " + c + ".");
+ }
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ final double relativeMem = 1.0/ this.memoryConsumerWeights;
+ c.setRelativeTempMemory(relativeMem);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
+ "table for " + c + ".");
+ }
+ }
+ }
+ }
+ }
+ return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
+ }
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ // if we come here again, prevent a further descend
+ if (!this.allNodes.add(visitable)) {
+ return false;
+ }
+
+ if (visitable instanceof SinkPlanNode) {
+ this.sinks.add((SinkPlanNode) visitable);
+ }
+ else if (visitable instanceof SourcePlanNode) {
+ this.sources.add((SourcePlanNode) visitable);
+ }
+ else if (visitable instanceof BinaryUnionPlanNode) {
+ BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ if (unionNode.unionsStaticAndDynamicPath()) {
+ unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
+ }
+ }
+ else if (visitable instanceof BulkPartialSolutionPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof WorksetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof SolutionSetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+
+ // double-connect the connections. previously, only parents knew their children, because
+ // one child candidate could have been referenced by multiple parents.
+ for (Channel conn : visitable.getInputs()) {
+ conn.setTarget(visitable);
+ conn.getSource().addOutgoingChannel(conn);
+ }
+
+ for (Channel c : visitable.getBroadcastInputs()) {
+ c.setTarget(visitable);
+ c.getSource().addOutgoingChannel(c);
+ }
+
+ // count the memory consumption
+ this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
+ for (Channel c : visitable.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+ for (Channel c : visitable.getBroadcastInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+
+ // pass the visitor to the iteraton's step function
+ if (visitable instanceof IterationPlanNode) {
+ // push the iteration node onto the stack
+ final IterationPlanNode iterNode = (IterationPlanNode) visitable;
+ this.stackOfIterationNodes.addLast(iterNode);
+
+ // recurse
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+
+ // pop the iteration node from the stack
+ this.stackOfIterationNodes.removeLast();
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {}
+ }
+
+ /**
+ * A visitor that traverses the graph and collects cascading binary unions into a single n-ary
+ * union operator. The exception is, when on of the union inputs is materialized, such as in the
+ * static-code-path-cache in iterations.
+ */
+ private static final class BinaryUnionReplacer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (this.seenBefore.add(visitable)) {
+ if (visitable instanceof IterationPlanNode) {
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+
+ if (visitable instanceof BinaryUnionPlanNode) {
+
+ final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ final Channel in1 = unionNode.getInput1();
+ final Channel in2 = unionNode.getInput2();
+
+ if (!unionNode.unionsStaticAndDynamicPath()) {
+
+ // both on static path, or both on dynamic path. we can collapse them
+ NAryUnionPlanNode newUnionNode;
+
+ List<Channel> inputs = new ArrayList<Channel>();
+ collect(in1, inputs);
+ collect(in2, inputs);
+
+ newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
+ unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
+
+ newUnionNode.setParallelism(unionNode.getParallelism());
+
+ for (Channel c : inputs) {
+ c.setTarget(newUnionNode);
+ }
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ else {
+ // union between the static and the dynamic path. we need to handle this for now
+ // through a special union operator
+
+ // make sure that the first input is the cached (static) and the second input is the dynamic
+ if (in1.isOnDynamicPath()) {
+ BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
+
+ in1.setTarget(newUnionNode);
+ in2.setTarget(newUnionNode);
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ }
+ }
+ }
+
+ private void collect(Channel in, List<Channel> inputs) {
+ if (in.getSource() instanceof NAryUnionPlanNode) {
+ // sanity check
+ if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
+ }
+ if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
+ }
+
+ inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
+ } else {
+ // is not a collapsed union node, so we take the channel directly
+ inputs.add(in);
+ }
+ }
+ }
+
+ private static final class StepFunctionValidator implements Visitor<Operator<?>> {
+
+ private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
+
+ private boolean foundWorkset;
+
+ @Override
+ public boolean preVisit(Operator<?> visitable) {
+ if (visitable instanceof WorksetPlaceHolder) {
+ foundWorkset = true;
+ }
+
+ return (!foundWorkset) && seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(Operator<?> visitable) {}
+ }
+
+ // ------------------------------------------------------------------------
+ // Miscellaneous
+ // ------------------------------------------------------------------------
+
+ private OptimizerPostPass getPostPassFromPlan(Plan program) {
+ final String className = program.getPostPassClassName();
+ if (className == null) {
+ throw new CompilerException("Optimizer Post Pass class description is null");
+ }
+ try {
+ Class<? extends OptimizerPostPass> clazz = Class.forName(className).asSubclass(OptimizerPostPass.class);
+ try {
+ return InstantiationUtil.instantiate(clazz, OptimizerPostPass.class);
+ } catch (RuntimeException rtex) {
+ // unwrap the source exception
+ if (rtex.getCause() != null) {
+ throw new CompilerException("Cannot instantiate optimizer post pass: " + rtex.getMessage(), rtex.getCause());
+ } else {
+ throw rtex;
+ }
+ }
+ } catch (ClassNotFoundException cnfex) {
+ throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex);
+ } catch (ClassCastException ccex) {
+ throw new CompilerException("Class '" + className + "' is not an optimizer post passer.", ccex);
+ }
+ }
+}
[41/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Removed
obsolete plan validator
Posted by se...@apache.org.
[FLINK-441] [optimizer] Removed obsolete plan validator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c91fb7c2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c91fb7c2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c91fb7c2
Branch: refs/heads/master
Commit: c91fb7c2f058272d8207a1d2e9396755c75cd853
Parents: 258cede
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 10:57:08 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
.../org/apache/flink/client/LocalExecutor.java | 4 -
.../org/apache/flink/client/program/Client.java | 5 +-
.../optimizer/contextcheck/ContextChecker.java | 211 -------------------
.../contextcheck/MissingChildException.java | 35 ---
.../optimizer/contextcheck/Validatable.java | 30 ---
.../mapred/record/HadoopDataSink.java | 13 +-
6 files changed, 2 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
index 46598c1..a2aed8f 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
@@ -34,7 +34,6 @@ import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.DataStatistics;
import org.apache.flink.optimizer.PactCompiler;
-import org.apache.flink.optimizer.contextcheck.ContextChecker;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
@@ -136,9 +135,6 @@ public class LocalExecutor extends PlanExecutor {
throw new IllegalArgumentException("The plan may not be null.");
}
- ContextChecker checker = new ContextChecker();
- checker.check(plan);
-
synchronized (this.lock) {
// check if we start a session dedicated for this execution
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
index ce77275..84e6637 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
@@ -33,7 +33,6 @@ import org.apache.flink.api.java.ExecutionEnvironmentFactory;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
import org.apache.flink.optimizer.PactCompiler;
-import org.apache.flink.optimizer.contextcheck.ContextChecker;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
import org.apache.flink.optimizer.plan.FlinkPlan;
import org.apache.flink.optimizer.plan.OptimizedPlan;
@@ -194,9 +193,7 @@ public class Client {
if (parallelism > 0 && p.getDefaultParallelism() <= 0) {
p.setDefaultParallelism(parallelism);
}
-
- ContextChecker checker = new ContextChecker();
- checker.check(p);
+
return this.compiler.compile(p);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/ContextChecker.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/ContextChecker.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/ContextChecker.java
deleted file mode 100644
index da73d36..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/ContextChecker.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.contextcheck;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.DualInputOperator;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.SingleInputOperator;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.base.FileDataSinkBase;
-import org.apache.flink.api.common.operators.base.FileDataSourceBase;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.util.Visitor;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Traverses a plan and checks whether all Contracts are correctly connected to
- * their input contracts.
- */
-public class ContextChecker implements Visitor<Operator<?>> {
-
- /**
- * A set of all already visited nodes during DAG traversal. Is used
- * to avoid processing one node multiple times.
- */
- private final Set<Operator<?>> visitedNodes = new HashSet<Operator<?>>();
-
- /**
- * Default constructor
- */
- public ContextChecker() {}
-
- /**
- * Checks whether the given plan is valid. In particular it is checked that
- * all contracts have the correct number of inputs and all inputs are of the
- * expected type. In case of an invalid plan an extended RuntimeException is
- * thrown.
- *
- * @param plan
- * The plan to check.
- */
- public void check(Plan plan) {
- Preconditions.checkNotNull(plan, "The passed plan is null.");
- this.visitedNodes.clear();
- plan.accept(this);
- }
-
- /**
- * Checks whether the node is correctly connected to its input.
- */
- @Override
- public boolean preVisit(Operator<?> node) {
- // check if node was already visited
- if (!this.visitedNodes.add(node)) {
- return false;
- }
-
- // apply the appropriate check method
- if (node instanceof FileDataSinkBase) {
- checkFileDataSink((FileDataSinkBase<?>) node);
- } else if (node instanceof FileDataSourceBase) {
- checkFileDataSource((FileDataSourceBase<?>) node);
- } else if (node instanceof GenericDataSinkBase) {
- checkDataSink((GenericDataSinkBase<?>) node);
- } else if (node instanceof BulkIterationBase) {
- checkBulkIteration((BulkIterationBase<?>) node);
- } else if (node instanceof SingleInputOperator) {
- checkSingleInputContract((SingleInputOperator<?, ?, ?>) node);
- } else if (node instanceof DualInputOperator<?, ?, ?, ?>) {
- checkDualInputContract((DualInputOperator<?, ?, ?, ?>) node);
- }
- if(node instanceof Validatable) {
- ((Validatable) node).check();
- }
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> node) {}
-
- /**
- * Checks if DataSinkContract is correctly connected. In case that the
- * contract is incorrectly connected a RuntimeException is thrown.
- *
- * @param dataSinkContract
- * DataSinkContract that is checked.
- */
- private void checkDataSink(GenericDataSinkBase<?> dataSinkContract) {
- Operator<?> input = dataSinkContract.getInput();
- // check if input exists
- if (input == null) {
- throw new MissingChildException();
- }
- }
-
- /**
- * Checks if FileDataSink is correctly connected. In case that the
- * contract is incorrectly connected a RuntimeException is thrown.
- *
- * @param fileSink
- * FileDataSink that is checked.
- */
- private void checkFileDataSink(FileDataSinkBase<?> fileSink) {
- String path = fileSink.getFilePath();
- if (path == null) {
- throw new InvalidProgramException("File path of FileDataSink is null.");
- }
- if (path.length() == 0) {
- throw new InvalidProgramException("File path of FileDataSink is empty string.");
- }
-
- try {
- Path p = new Path(path);
- String scheme = p.toUri().getScheme();
-
- if (scheme == null) {
- throw new InvalidProgramException("File path \"" + path + "\" of FileDataSink has no file system scheme (like 'file:// or hdfs://').");
- }
- } catch (Exception e) {
- throw new InvalidProgramException("File path \"" + path + "\" of FileDataSink is an invalid path: " + e.getMessage());
- }
- checkDataSink(fileSink);
- }
-
- /**
- * Checks if FileDataSource is correctly connected. In case that the
- * contract is incorrectly connected a RuntimeException is thrown.
- *
- * @param fileSource
- * FileDataSource that is checked.
- */
- private void checkFileDataSource(FileDataSourceBase<?> fileSource) {
- String path = fileSource.getFilePath();
- if (path == null) {
- throw new InvalidProgramException("File path of FileDataSource is null.");
- }
- if (path.length() == 0) {
- throw new InvalidProgramException("File path of FileDataSource is empty string.");
- }
-
- try {
- Path p = new Path(path);
- String scheme = p.toUri().getScheme();
-
- if (scheme == null) {
- throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource has no file system scheme (like 'file:// or hdfs://').");
- }
- } catch (Exception e) {
- throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource is an invalid path: " + e.getMessage());
- }
- }
-
- /**
- * Checks whether a SingleInputOperator is correctly connected. In case that
- * the contract is incorrectly connected a RuntimeException is thrown.
- *
- * @param singleInputContract
- * SingleInputOperator that is checked.
- */
- private void checkSingleInputContract(SingleInputOperator<?, ?, ?> singleInputContract) {
- Operator<?> input = singleInputContract.getInput();
- // check if input exists
- if (input == null) {
- throw new MissingChildException();
- }
- }
-
- /**
- * Checks whether a DualInputOperator is correctly connected. In case that
- * the contract is incorrectly connected a RuntimeException is thrown.
- *
- * @param dualInputContract
- * DualInputOperator that is checked.
- */
- private void checkDualInputContract(DualInputOperator<?, ?, ?, ?> dualInputContract) {
- Operator<?> input1 = dualInputContract.getFirstInput();
- Operator<?> input2 = dualInputContract.getSecondInput();
- // check if input exists
- if (input1 == null || input2 == null) {
- throw new MissingChildException();
- }
- }
-
- private void checkBulkIteration(BulkIterationBase<?> iter) {
- iter.validate();
- checkSingleInputContract(iter);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/MissingChildException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/MissingChildException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/MissingChildException.java
deleted file mode 100644
index 04b2b4c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/MissingChildException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.contextcheck;
-
-import org.apache.flink.api.common.InvalidProgramException;
-
-public class MissingChildException extends InvalidProgramException {
-
- private static final long serialVersionUID = 4206417538759568484L;
-
- public MissingChildException(String message) {
- super(message);
- }
-
- public MissingChildException() {
- super();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/Validatable.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/Validatable.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/Validatable.java
deleted file mode 100644
index 161bd66..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/contextcheck/Validatable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.contextcheck;
-
-/**
- * Operators implementing this interface
- * will be called from the {@link ContextChecker} during
- * the compilation process.
- *
- */
-public interface Validatable {
- public void check();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c91fb7c2/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
index a86cc62..3b8064f 100644
--- a/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
+++ b/flink-staging/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/HadoopDataSink.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.hadoopcompatibility.mapred.record;
import java.util.Collections;
@@ -24,11 +23,9 @@ import java.util.List;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.java.record.operators.GenericDataSink;
-import org.apache.flink.optimizer.contextcheck.Validatable;
import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter;
import org.apache.flink.hadoopcompatibility.mapred.record.datatypes.FlinkTypeConverter;
import org.apache.flink.types.Record;
-import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputFormat;
@@ -45,7 +42,7 @@ import org.apache.hadoop.mapred.OutputFormat;
*
* The HadoopDataSink provides a default converter: {@link org.apache.flink.hadoopcompatibility.mapred.record.datatypes.DefaultFlinkTypeConverter}
**/
-public class HadoopDataSink<K,V> extends GenericDataSink implements Validatable {
+public class HadoopDataSink<K,V> extends GenericDataSink {
private static String DEFAULT_NAME = "<Unnamed Hadoop Data Sink>";
@@ -98,12 +95,4 @@ public class HadoopDataSink<K,V> extends GenericDataSink implements Validatable
public JobConf getJobConf() {
return this.jobConf;
}
-
- @Override
- public void check() {
- // see for more details https://github.com/stratosphere/stratosphere/pull/531
- if (FileOutputFormat.getOutputPath(jobConf) == null) {
- throw new NullPointerException("The HadoopDataSink currently expects a correct outputPath.");
- }
- }
}
[46/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/IterativeKMeansTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/IterativeKMeansTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/IterativeKMeansTest.java
index 5a0a6fc..bf2bbae 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/IterativeKMeansTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/IterativeKMeansTest.java
@@ -32,7 +32,7 @@ import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
@@ -77,7 +77,7 @@ public class IterativeKMeansTest extends CompilerTestBase {
OptimizedPlan plan = compileWithStats(p);
checkPlan(plan);
- new NepheleJobGraphGenerator().compileJobGraph(plan);
+ new JobGraphGenerator().compileJobGraph(plan);
}
@Test
@@ -89,7 +89,7 @@ public class IterativeKMeansTest extends CompilerTestBase {
OptimizedPlan plan = compileNoStats(p);
checkPlan(plan);
- new NepheleJobGraphGenerator().compileJobGraph(plan);
+ new JobGraphGenerator().compileJobGraph(plan);
}
private void checkPlan(OptimizedPlan plan) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
index cc8f788..e6a1e69 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
@@ -39,7 +39,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
@SuppressWarnings("serial")
public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
@@ -79,7 +79,7 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
assertEquals(SolutionSetPlanNode.class, join1.getInput1().getSource().getClass());
assertEquals(SolutionSetPlanNode.class, join2.getInput2().getSource().getClass());
- new NepheleJobGraphGenerator().compileJobGraph(optPlan);
+ new JobGraphGenerator().compileJobGraph(optPlan);
}
catch (Exception e) {
System.err.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/PageRankCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/PageRankCompilerTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/PageRankCompilerTest.java
index 731e344..f05bd25 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/PageRankCompilerTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/PageRankCompilerTest.java
@@ -24,7 +24,7 @@ import static org.junit.Assert.fail;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
@@ -68,7 +68,7 @@ public class PageRankCompilerTest extends CompilerTestBase{
IterativeDataSet<Tuple2<Long, Double>> iteration = pagesWithRanks.iterate(10);
Configuration cfg = new Configuration();
- cfg.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
+ cfg.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
DataSet<Tuple2<Long, Double>> newRanks = iteration
// join pages with outgoing edges and distribute rank
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/compiler/plandump/PreviewPlanDumpTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/plandump/PreviewPlanDumpTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/plandump/PreviewPlanDumpTest.java
index 1cf92e6..b348333 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/plandump/PreviewPlanDumpTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/plandump/PreviewPlanDumpTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.test.compiler.plandump;
import java.util.List;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.test.recordJobs.graph.DeltaPageRankWithInitialDeltas;
@@ -89,7 +89,7 @@ public class PreviewPlanDumpTest {
private void dump(Plan p) {
try {
- List<DataSinkNode> sinks = PactCompiler.createPreOptimizedPlan(p);
+ List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(p);
PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
String json = dumper.getPactPlanAsJSON(sinks);
JsonParser parser = new JsonFactory().createJsonParser(json);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountITCase.java
index b88eb4e..41b84e6 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountITCase.java
@@ -27,11 +27,6 @@ public class WordCountITCase extends JavaProgramTestBase {
protected String textPath;
protected String resultPath;
- public WordCountITCase(){
- setDegreeOfParallelism(4);
- setNumTaskManagers(2);
- setTaskManagerNumSlots(2);
- }
@Override
protected void preSubmit() throws Exception {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/TaskFailureITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/TaskFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/TaskFailureITCase.java
index a5c2bb4..9021c6a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/TaskFailureITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/failingPrograms/TaskFailureITCase.java
@@ -24,9 +24,9 @@ import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.MapOperator;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseInputFormat;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseOutputFormat;
@@ -88,11 +88,11 @@ public class TaskFailureITCase extends FailingTestBase {
plan.setDefaultParallelism(DOP);
// optimize and compile plan
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
// return job graph of failing job
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
@@ -118,11 +118,11 @@ public class TaskFailureITCase extends FailingTestBase {
plan.setDefaultParallelism(4);
// optimize and compile plan
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
// return job graph of working job
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
index 037610e..a0e3468 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
@@ -39,12 +39,6 @@ public class WordCountMapredITCase extends JavaProgramTestBase {
protected String textPath;
protected String resultPath;
- public WordCountMapredITCase(){
-// setDegreeOfParallelism(4);
-// setNumTaskManagers(2);
-// setTaskManagerNumSlots(2);
- }
-
@Override
protected void preSubmit() throws Exception {
textPath = createTempFile("text.txt", WordCountData.TEXT);
@@ -59,7 +53,6 @@ public class WordCountMapredITCase extends JavaProgramTestBase {
@Override
protected void testProgram() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-// env.setDegreeOfParallelism(1);
DataSet<Tuple2<LongWritable, Text>> input = env.readHadoopFile(new TextInputFormat(),
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
index 3bdaa22..fee49bf 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
@@ -39,12 +39,6 @@ public class WordCountMapreduceITCase extends JavaProgramTestBase {
protected String textPath;
protected String resultPath;
- public WordCountMapreduceITCase(){
-// setDegreeOfParallelism(4);
-// setNumTaskManagers(2);
-// setTaskManagerNumSlots(2);
- }
-
@Override
protected void preSubmit() throws Exception {
textPath = createTempFile("text.txt", WordCountData.TEXT);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index 79d49aa..e5f91b4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -31,7 +31,7 @@ import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CrazyNested;
@@ -377,7 +377,7 @@ public class GroupReduceITCase extends MultipleProgramsTestBase {
.map(new IdentityMapper<Tuple3<Integer, Long, String>>()).setParallelism(4);
Configuration cfg = new Configuration();
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION);
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION);
DataSet<Tuple2<Integer, String>> reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine())
.withParameters(cfg);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
index d92897d..220611d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
@@ -32,7 +32,7 @@ import org.apache.flink.api.java.record.io.FileOutputFormat;
import org.apache.flink.api.java.record.operators.CoGroupOperator;
import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.RecordAPITestBase;
import org.apache.flink.types.IntValue;
@@ -156,9 +156,9 @@ public class CoGroupITCase extends RecordAPITestBase {
CoGroupOperator testCoGrouper = CoGroupOperator.builder(new TestCoGrouper(), StringValue.class, 0, 0)
.build();
testCoGrouper.setDegreeOfParallelism(config.getInteger("CoGroupTest#NoSubtasks", 1));
- testCoGrouper.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
+ testCoGrouper.getParameters().setString(Optimizer.HINT_LOCAL_STRATEGY,
config.getString("CoGroupTest#LocalStrategy", ""));
- testCoGrouper.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
+ testCoGrouper.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY,
config.getString("CoGroupTest#ShipStrategy", ""));
FileDataSink output = new FileDataSink(new CoGroupOutFormat(), resultPath);
@@ -181,9 +181,9 @@ public class CoGroupITCase extends RecordAPITestBase {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
- String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE };
+ String[] localStrategies = { Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE };
- String[] shipStrategies = { PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH };
+ String[] shipStrategies = { Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH };
for (String localStrategy : localStrategies) {
for (String shipStrategy : shipStrategies) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
index f72d146..f6b4127 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
@@ -24,7 +24,7 @@ import org.apache.flink.api.java.record.io.DelimitedInputFormat;
import org.apache.flink.api.java.record.operators.CrossOperator;
import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseInputFormat;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseOutputFormat;
@@ -132,20 +132,20 @@ public class CrossITCase extends RecordAPITestBase {
CrossOperator testCross = CrossOperator.builder(new TestCross()).build();
testCross.setDegreeOfParallelism(config.getInteger("CrossTest#NoSubtasks", 1));
- testCross.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
+ testCross.getParameters().setString(Optimizer.HINT_LOCAL_STRATEGY,
config.getString("CrossTest#LocalStrategy", ""));
if (config.getString("CrossTest#ShipStrategy", "").equals("BROADCAST_FIRST")) {
- testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
- testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
+ testCross.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_BROADCAST);
+ testCross.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_FORWARD);
} else if (config.getString("CrossTest#ShipStrategy", "").equals("BROADCAST_SECOND")) {
- testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
- testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
+ testCross.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_BROADCAST);
+ testCross.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_FORWARD);
} else {
- testCross.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
+ testCross.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY,
config.getString("CrossTest#ShipStrategy", ""));
}
@@ -170,10 +170,10 @@ public class CrossITCase extends RecordAPITestBase {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
- String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST,
- PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND,
- PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST,
- PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND };
+ String[] localStrategies = { Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST,
+ Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND,
+ Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST,
+ Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND };
String[] shipStrategies = { "BROADCAST_FIRST", "BROADCAST_SECOND"
// PactCompiler.HINT_SHIP_STRATEGY_BROADCAST
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/operators/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/JoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/JoinITCase.java
index 6906d36..02a6e38 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/JoinITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/JoinITCase.java
@@ -27,7 +27,7 @@ import org.apache.flink.api.java.record.io.DelimitedInputFormat;
import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.JoinOperator;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseInputFormat;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseOutputFormat;
@@ -122,20 +122,20 @@ public class JoinITCase extends RecordAPITestBase {
JoinOperator testMatcher = JoinOperator.builder(new TestMatcher(), StringValue.class, 0, 0)
.build();
testMatcher.setDegreeOfParallelism(config.getInteger("MatchTest#NoSubtasks", 1));
- testMatcher.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
+ testMatcher.getParameters().setString(Optimizer.HINT_LOCAL_STRATEGY,
config.getString("MatchTest#LocalStrategy", ""));
if (config.getString("MatchTest#ShipStrategy", "").equals("BROADCAST_FIRST")) {
- testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
- testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
+ testMatcher.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_BROADCAST);
+ testMatcher.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_FORWARD);
} else if (config.getString("MatchTest#ShipStrategy", "").equals("BROADCAST_SECOND")) {
- testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_FORWARD);
- testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT,
- PactCompiler.HINT_SHIP_STRATEGY_BROADCAST);
+ testMatcher.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_FORWARD);
+ testMatcher.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT,
+ Optimizer.HINT_SHIP_STRATEGY_BROADCAST);
} else {
- testMatcher.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
+ testMatcher.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY,
config.getString("MatchTest#ShipStrategy", ""));
}
@@ -160,10 +160,10 @@ public class JoinITCase extends RecordAPITestBase {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
- String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE,
- PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST, PactCompiler.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND };
+ String[] localStrategies = { Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE,
+ Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST, Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND };
- String[] shipStrategies = { PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH, "BROADCAST_FIRST", "BROADCAST_SECOND"};
+ String[] shipStrategies = { Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH, "BROADCAST_FIRST", "BROADCAST_SECOND"};
for (String localStrategy : localStrategies) {
for (String shipStrategy : shipStrategies) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
index 5f5f33d..3c8d372 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
@@ -25,9 +25,9 @@ import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.ReduceOperator;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseInputFormat;
@@ -119,9 +119,9 @@ public class ReduceITCase extends RecordAPITestBase {
ReduceOperator testReducer = ReduceOperator.builder(new TestReducer(), StringValue.class, 0)
.build();
testReducer.setDegreeOfParallelism(config.getInteger("ReduceTest#NoSubtasks", 1));
- testReducer.getParameters().setString(PactCompiler.HINT_LOCAL_STRATEGY,
+ testReducer.getParameters().setString(Optimizer.HINT_LOCAL_STRATEGY,
config.getString("ReduceTest#LocalStrategy", ""));
- testReducer.getParameters().setString(PactCompiler.HINT_SHIP_STRATEGY,
+ testReducer.getParameters().setString(Optimizer.HINT_SHIP_STRATEGY,
config.getString("ReduceTest#ShipStrategy", ""));
FileDataSink output = new FileDataSink(
@@ -133,10 +133,10 @@ public class ReduceITCase extends RecordAPITestBase {
Plan plan = new Plan(output);
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
@@ -151,8 +151,8 @@ public class ReduceITCase extends RecordAPITestBase {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
- String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_SORT };
- String[] shipStrategies = { PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH };
+ String[] localStrategies = { Optimizer.HINT_LOCAL_STRATEGY_SORT };
+ String[] shipStrategies = { Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH };
for (String localStrategy : localStrategies) {
for (String shipStrategy : shipStrategies) {
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/java/org/apache/flink/test/operators/UnionSinkITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/UnionSinkITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/UnionSinkITCase.java
index 4b9bc2a..944de98 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/UnionSinkITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/UnionSinkITCase.java
@@ -28,9 +28,9 @@ import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.MapOperator;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.NepheleJobGraphGenerator;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.test.operators.io.ContractITCaseIOFormats.ContractITCaseInputFormat;
@@ -124,10 +124,10 @@ public class UnionSinkITCase extends RecordAPITestBase {
Plan plan = new Plan(output);
plan.setDefaultParallelism(DOP);
- PactCompiler pc = new PactCompiler(new DataStatistics());
+ Optimizer pc = new Optimizer(new DataStatistics());
OptimizedPlan op = pc.compile(plan);
- NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
+ JobGraphGenerator jgg = new JobGraphGenerator();
return jgg.compileJobGraph(op);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
index 853a477..33e0807 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
@@ -25,7 +25,7 @@ import org.apache.flink.api.scala.ExecutionEnvironment
import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.api.scala.util.CollectionDataSets.{CrazyNested, POJO, MutableTuple3,
CustomType}
-import org.apache.flink.optimizer.PactCompiler
+import org.apache.flink.optimizer.Optimizer
import org.apache.flink.configuration.Configuration
import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
@@ -324,7 +324,7 @@ class GroupReduceITCase(mode: TestExecutionMode) extends MultipleProgramsTestBas
val ds = CollectionDataSets.get3TupleDataSet(env).map(t => t).setParallelism(4)
val cfg: Configuration = new Configuration
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)
+ cfg.setString(Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION)
val reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine).withParameters(cfg)
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CustomPartitioningTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CustomPartitioningTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CustomPartitioningTest.scala
index 8ac52cb..969f970 100644
--- a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CustomPartitioningTest.scala
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/CustomPartitioningTest.scala
@@ -54,17 +54,17 @@ class CustomPartitioningTest extends CompilerTestBase {
val balancer = partitioner.getInput.getSource.asInstanceOf[SingleInputPlanNode]
assertEquals(ShipStrategyType.FORWARD, sink.getInput.getShipStrategy)
- assertEquals(parallelism, sink.getDegreeOfParallelism)
+ assertEquals(parallelism, sink.getParallelism)
assertEquals(ShipStrategyType.FORWARD, mapper.getInput.getShipStrategy)
- assertEquals(parallelism, mapper.getDegreeOfParallelism)
+ assertEquals(parallelism, mapper.getParallelism)
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput.getShipStrategy)
assertEquals(part, partitioner.getInput.getPartitioner)
- assertEquals(parallelism, partitioner.getDegreeOfParallelism)
+ assertEquals(parallelism, partitioner.getParallelism)
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput.getShipStrategy)
- assertEquals(parallelism, balancer.getDegreeOfParallelism)
+ assertEquals(parallelism, balancer.getParallelism)
}
catch {
case e: Exception => {
@@ -124,17 +124,17 @@ class CustomPartitioningTest extends CompilerTestBase {
val balancer = partitioner.getInput.getSource.asInstanceOf[SingleInputPlanNode]
assertEquals(ShipStrategyType.FORWARD, sink.getInput.getShipStrategy)
- assertEquals(parallelism, sink.getDegreeOfParallelism)
+ assertEquals(parallelism, sink.getParallelism)
assertEquals(ShipStrategyType.FORWARD, mapper.getInput.getShipStrategy)
- assertEquals(parallelism, mapper.getDegreeOfParallelism)
+ assertEquals(parallelism, mapper.getParallelism)
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput.getShipStrategy)
assertEquals(part, partitioner.getInput.getPartitioner)
- assertEquals(parallelism, partitioner.getDegreeOfParallelism)
+ assertEquals(parallelism, partitioner.getParallelism)
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput.getShipStrategy)
- assertEquals(parallelism, balancer.getDegreeOfParallelism)
+ assertEquals(parallelism, balancer.getParallelism)
}
catch {
case e: Exception => {
@@ -197,23 +197,23 @@ class CustomPartitioningTest extends CompilerTestBase {
val balancer = keyExtractor.getInput.getSource.asInstanceOf[SingleInputPlanNode]
assertEquals(ShipStrategyType.FORWARD, sink.getInput.getShipStrategy)
- assertEquals(parallelism, sink.getDegreeOfParallelism)
+ assertEquals(parallelism, sink.getParallelism)
assertEquals(ShipStrategyType.FORWARD, mapper.getInput.getShipStrategy)
- assertEquals(parallelism, mapper.getDegreeOfParallelism)
+ assertEquals(parallelism, mapper.getParallelism)
assertEquals(ShipStrategyType.FORWARD, keyRemover.getInput.getShipStrategy)
- assertEquals(parallelism, keyRemover.getDegreeOfParallelism)
+ assertEquals(parallelism, keyRemover.getParallelism)
assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput.getShipStrategy)
assertEquals(part, partitioner.getInput.getPartitioner)
- assertEquals(parallelism, partitioner.getDegreeOfParallelism)
+ assertEquals(parallelism, partitioner.getParallelism)
assertEquals(ShipStrategyType.FORWARD, keyExtractor.getInput.getShipStrategy)
- assertEquals(parallelism, keyExtractor.getDegreeOfParallelism)
+ assertEquals(parallelism, keyExtractor.getParallelism)
assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput.getShipStrategy)
- assertEquals(parallelism, balancer.getDegreeOfParallelism)
+ assertEquals(parallelism, balancer.getParallelism)
}
catch {
case e: Exception => {
[26/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
deleted file mode 100644
index e65758f..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/NestedIterationsTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.junit.Test;
-
-@SuppressWarnings({"serial", "unchecked"})
-public class NestedIterationsTest extends CompilerTestBase {
-
- @Test
- public void testRejectNestedBulkIterations() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> data = env.generateSequence(1, 100);
-
- IterativeDataSet<Long> outerIteration = data.iterate(100);
-
- IterativeDataSet<Long> innerIteration = outerIteration.map(new IdentityMapper<Long>()).iterate(100);
-
- DataSet<Long> innerResult = innerIteration.closeWith(innerIteration.map(new IdentityMapper<Long>()));
-
- DataSet<Long> outerResult = outerIteration.closeWith(innerResult.map(new IdentityMapper<Long>()));
-
- outerResult.print();
-
- Plan p = env.createProgramPlan();
-
- try {
- compileNoStats(p);
- }
- catch (CompilerException e) {
- assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testRejectNestedWorksetIterations() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> data = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> outerIteration = data.iterateDelta(data, 100, 0);
-
- DataSet<Tuple2<Long, Long>> inOuter = outerIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> innerIteration = inOuter.iterateDelta(inOuter, 100, 0);
-
- DataSet<Tuple2<Long, Long>> inInner = innerIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> innerResult = innerIteration.closeWith(inInner, inInner).map(new IdentityMapper<Tuple2<Long,Long>>());
-
- DataSet<Tuple2<Long, Long>> outerResult = outerIteration.closeWith(innerResult, innerResult);
-
- outerResult.print();
-
- Plan p = env.createProgramPlan();
-
- try {
- compileNoStats(p);
- }
- catch (CompilerException e) {
- assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testBulkIterationInClosure() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Long> data1 = env.generateSequence(1, 100);
- DataSet<Long> data2 = env.generateSequence(1, 100);
-
- IterativeDataSet<Long> firstIteration = data1.iterate(100);
-
- DataSet<Long> firstResult = firstIteration.closeWith(firstIteration.map(new IdentityMapper<Long>()));
-
-
- IterativeDataSet<Long> mainIteration = data2.map(new IdentityMapper<Long>()).iterate(100);
-
- DataSet<Long> joined = mainIteration.join(firstResult)
- .where(new IdentityKeyExtractor<Long>()).equalTo(new IdentityKeyExtractor<Long>())
- .with(new DummyFlatJoinFunction<Long>());
-
- DataSet<Long> mainResult = mainIteration.closeWith(joined);
-
- mainResult.print();
-
- Plan p = env.createProgramPlan();
-
- // optimizer should be able to translate this
- OptimizedPlan op = compileNoStats(p);
-
- // job graph generator should be able to translate this
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testDeltaIterationInClosure() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> data1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
- DataSet<Tuple2<Long, Long>> data2 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> firstIteration = data1.iterateDelta(data1, 100, 0);
-
- DataSet<Tuple2<Long, Long>> inFirst = firstIteration.getWorkset().map(new IdentityMapper<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> firstResult = firstIteration.closeWith(inFirst, inFirst).map(new IdentityMapper<Tuple2<Long,Long>>());
-
-
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> mainIteration = data2.iterateDelta(data2, 100, 0);
-
- DataSet<Tuple2<Long, Long>> joined = mainIteration.getWorkset().join(firstResult).where(0).equalTo(0)
- .projectFirst(0).projectSecond(0);
-
- DataSet<Tuple2<Long, Long>> mainResult = mainIteration.closeWith(joined, joined);
-
- mainResult.print();
-
- Plan p = env.createProgramPlan();
-
- // optimizer should be able to translate this
- OptimizedPlan op = compileNoStats(p);
-
- // job graph generator should be able to translate this
- new JobGraphGenerator().compileJobGraph(op);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
deleted file mode 100644
index 2b42f85..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitionPushdownTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.junit.Test;
-
-@SuppressWarnings("serial")
-public class PartitionPushdownTest extends CompilerTestBase {
-
- @Test
- public void testPartitioningNotPushedDown() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input
- .groupBy(0, 1).sum(2)
- .groupBy(0).sum(1)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
-
- SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
- SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
-
- assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
- assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
-
- assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy());
-
- assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
- assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPartitioningReused() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings("unchecked")
- DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
-
- input
- .groupBy(0).sum(1)
- .groupBy(0, 1).sum(2)
- .print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
-
- SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
- SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
-
- assertEquals(ShipStrategyType.FORWARD, agg2Reducer.getInput().getShipStrategy());
-
- assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
- assertEquals(new FieldList(0), agg1Reducer.getInput().getShipStrategyKeys());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
deleted file mode 100644
index 16684dc..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/PartitioningReusageTest.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.functions.CoGroupFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.translation.JavaPlan;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-@SuppressWarnings("serial")
-public class PartitioningReusageTest extends CompilerTestBase {
-
- @Test
- public void noPreviousPartitioningJoin1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(0,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
-
- }
-
- @Test
- public void noPreviousPartitioningJoin2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
-
- }
-
- @Test
- public void reuseSinglePartitioningJoin1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(0,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
-
- }
-
- @Test
- public void reuseSinglePartitioningJoin2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseSinglePartitioningJoin3() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .join(set2.partitionByHash(2, 1)
- .map(new MockMapper())
- .withForwardedFields("2;1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseSinglePartitioningJoin4() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0)
- .map(new MockMapper()).withForwardedFields("0")
- .join(set2, JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseSinglePartitioningJoin5() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .join(set2.partitionByHash(2)
- .map(new MockMapper())
- .withForwardedFields("2"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .join(set2.partitionByHash(0,1)
- .map(new MockMapper())
- .withForwardedFields("0;1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(0,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
-
- @Test
- public void reuseBothPartitioningJoin2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .join(set2.partitionByHash(1,2)
- .map(new MockMapper())
- .withForwardedFields("1;2"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin3() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0)
- .map(new MockMapper()).withForwardedFields("0")
- .join(set2.partitionByHash(2,1)
- .map(new MockMapper())
- .withForwardedFields("2;1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,1).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin4() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0,2)
- .map(new MockMapper()).withForwardedFields("0;2")
- .join(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,2).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin5() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(2)
- .map(new MockMapper()).withForwardedFields("2")
- .join(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,2).equalTo(2,1).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin6() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(0)
- .map(new MockMapper()).withForwardedFields("0")
- .join(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,2).equalTo(1,2).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
- @Test
- public void reuseBothPartitioningJoin7() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> joined = set1
- .partitionByHash(2)
- .map(new MockMapper()).withForwardedFields("2")
- .join(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"),
- JoinOperatorBase.JoinHint.REPARTITION_HASH_FIRST)
- .where(0,2).equalTo(1,2).with(new MockJoin());
-
- joined.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidJoinInputProperties(join);
- }
-
-
- @Test
- public void noPreviousPartitioningCoGroup1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .coGroup(set2)
- .where(0,1).equalTo(0,1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
-
- }
-
- @Test
- public void noPreviousPartitioningCoGroup2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .coGroup(set2)
- .where(0,1).equalTo(2,1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
-
- }
-
- @Test
- public void reuseSinglePartitioningCoGroup1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .coGroup(set2)
- .where(0,1).equalTo(0,1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
-
- }
-
- @Test
- public void reuseSinglePartitioningCoGroup2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .coGroup(set2)
- .where(0,1).equalTo(2,1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseSinglePartitioningCoGroup3() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .coGroup(set2.partitionByHash(2, 1)
- .map(new MockMapper())
- .withForwardedFields("2;1"))
- .where(0,1).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseSinglePartitioningCoGroup4() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0)
- .map(new MockMapper()).withForwardedFields("0")
- .coGroup(set2)
- .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseSinglePartitioningCoGroup5() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .coGroup(set2.partitionByHash(2)
- .map(new MockMapper())
- .withForwardedFields("2"))
- .where(0,1).equalTo(2,1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup1() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .coGroup(set2.partitionByHash(0, 1)
- .map(new MockMapper())
- .withForwardedFields("0;1"))
- .where(0, 1).equalTo(0, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
-
- @Test
- public void reuseBothPartitioningCoGroup2() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0,1)
- .map(new MockMapper()).withForwardedFields("0;1")
- .coGroup(set2.partitionByHash(1, 2)
- .map(new MockMapper())
- .withForwardedFields("1;2"))
- .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup3() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0)
- .map(new MockMapper()).withForwardedFields("0")
- .coGroup(set2.partitionByHash(2, 1)
- .map(new MockMapper())
- .withForwardedFields("2;1"))
- .where(0, 1).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup4() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(0,2)
- .map(new MockMapper()).withForwardedFields("0;2")
- .coGroup(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"))
- .where(0, 2).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup5() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(2)
- .map(new MockMapper()).withForwardedFields("2")
- .coGroup(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"))
- .where(0, 2).equalTo(2, 1).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup6() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(2)
- .map(new MockMapper()).withForwardedFields("2")
- .coGroup(set2.partitionByHash(2)
- .map(new MockMapper())
- .withForwardedFields("2"))
- .where(0, 2).equalTo(1, 2).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
- @Test
- public void reuseBothPartitioningCoGroup7() {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
- DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
-
- DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = set1
- .partitionByHash(2)
- .map(new MockMapper()).withForwardedFields("2")
- .coGroup(set2.partitionByHash(1)
- .map(new MockMapper())
- .withForwardedFields("1"))
- .where(0, 2).equalTo(1, 2).with(new MockCoGroup());
-
- coGrouped.print();
- JavaPlan plan = env.createProgramPlan();
- OptimizedPlan oPlan = compileWithStats(plan);
-
- SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
- DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
-
- checkValidCoGroupInputProperties(coGroup);
- }
-
-
-
- private void checkValidJoinInputProperties(DualInputPlanNode join) {
-
- GlobalProperties inProps1 = join.getInput1().getGlobalProperties();
- GlobalProperties inProps2 = join.getInput2().getGlobalProperties();
-
- if(inProps1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- inProps2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
-
- // check that both inputs are hash partitioned on the same fields
- FieldList pFields1 = inProps1.getPartitioningFields();
- FieldList pFields2 = inProps2.getPartitioningFields();
-
- assertTrue("Inputs are not the same number of fields. Input 1: "+pFields1+", Input 2: "+pFields2,
- pFields1.size() == pFields2.size());
-
- FieldList reqPFields1 = join.getKeysForInput1();
- FieldList reqPFields2 = join.getKeysForInput2();
-
- for(int i=0; i<pFields1.size(); i++) {
-
- // get fields
- int f1 = pFields1.get(i);
- int f2 = pFields2.get(i);
-
- // check that field positions in original key field list are identical
- int pos1 = getPosInFieldList(f1, reqPFields1);
- int pos2 = getPosInFieldList(f2, reqPFields2);
-
- if(pos1 < 0) {
- fail("Input 1 is partitioned on field "+f1+" which is not contained in the key set "+reqPFields1);
- }
- if(pos2 < 0) {
- fail("Input 2 is partitioned on field "+f2+" which is not contained in the key set "+reqPFields2);
- }
- if(pos1 != pos2) {
- fail("Inputs are not partitioned on the same key fields");
- }
- }
-
- }
- else if(inProps1.getPartitioning() == PartitioningProperty.FULL_REPLICATION &&
- inProps2.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED) {
- // we are good. No need to check for fields
- }
- else if(inProps1.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED &&
- inProps2.getPartitioning() == PartitioningProperty.FULL_REPLICATION) {
- // we are good. No need to check for fields
- }
- else {
- throw new UnsupportedOperationException("This method has only been implemented to check for hash partitioned coGroupinputs");
- }
-
- }
-
- private void checkValidCoGroupInputProperties(DualInputPlanNode coGroup) {
-
- GlobalProperties inProps1 = coGroup.getInput1().getGlobalProperties();
- GlobalProperties inProps2 = coGroup.getInput2().getGlobalProperties();
-
- if(inProps1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- inProps2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
-
- // check that both inputs are hash partitioned on the same fields
- FieldList pFields1 = inProps1.getPartitioningFields();
- FieldList pFields2 = inProps2.getPartitioningFields();
-
- assertTrue("Inputs are not the same number of fields. Input 1: "+pFields1+", Input 2: "+pFields2,
- pFields1.size() == pFields2.size());
-
- FieldList reqPFields1 = coGroup.getKeysForInput1();
- FieldList reqPFields2 = coGroup.getKeysForInput2();
-
- for(int i=0; i<pFields1.size(); i++) {
-
- // get fields
- int f1 = pFields1.get(i);
- int f2 = pFields2.get(i);
-
- // check that field positions in original key field list are identical
- int pos1 = getPosInFieldList(f1, reqPFields1);
- int pos2 = getPosInFieldList(f2, reqPFields2);
-
- if(pos1 < 0) {
- fail("Input 1 is partitioned on field "+f1+" which is not contained in the key set "+reqPFields1);
- }
- if(pos2 < 0) {
- fail("Input 2 is partitioned on field "+f2+" which is not contained in the key set "+reqPFields2);
- }
- if(pos1 != pos2) {
- fail("Inputs are not partitioned on the same key fields");
- }
- }
-
- }
- else {
- throw new UnsupportedOperationException("This method has only been implemented to check for hash partitioned coGroup inputs");
- }
-
- }
-
- private int getPosInFieldList(int field, FieldList list) {
-
- int pos;
- for(pos=0; pos<list.size(); pos++) {
- if(field == list.get(pos)) {
- break;
- }
- }
- if(pos == list.size()) {
- return -1;
- } else {
- return pos;
- }
-
- }
-
-
-
- public static class MockMapper implements MapFunction<Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
- @Override
- public Tuple3<Integer, Integer, Integer> map(Tuple3<Integer, Integer, Integer> value) throws Exception {
- return null;
- }
- }
-
- public static class MockJoin implements JoinFunction<Tuple3<Integer, Integer, Integer>,
- Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
-
- @Override
- public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
- return null;
- }
- }
-
- public static class MockCoGroup implements CoGroupFunction<Tuple3<Integer, Integer, Integer>,
- Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
-
- @Override
- public void coGroup(Iterable<Tuple3<Integer, Integer, Integer>> first, Iterable<Tuple3<Integer, Integer, Integer>> second,
- Collector<Tuple3<Integer, Integer, Integer>> out) throws Exception {
-
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
deleted file mode 100644
index 86f01b0..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/PipelineBreakerTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.optimizer.testfunctions.IdentityMapper;
-import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
-import org.junit.Test;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.configuration.Configuration;
-
-@SuppressWarnings("serial")
-public class PipelineBreakerTest extends CompilerTestBase {
-
- @Test
- public void testPipelineBreakerWithBroadcastVariable() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
-
- DataSet<Long> result = source.map(new IdentityMapper<Long>())
- .map(new IdentityMapper<Long>())
- .withBroadcastSet(source, "bc");
-
- result.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput().getTempMode().breaksPipeline());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPipelineBreakerBroadcastedAllReduce() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
-
- DataSet<Long> bcInput1 = sourceWithMapper
- .map(new IdentityMapper<Long>())
- .reduce(new SelectOneReducer<Long>());
- DataSet<Long> bcInput2 = env.generateSequence(1, 10);
-
- DataSet<Long> result = sourceWithMapper
- .map(new IdentityMapper<Long>())
- .withBroadcastSet(bcInput1, "bc1")
- .withBroadcastSet(bcInput2, "bc2");
-
- result.print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput().getTempMode().breaksPipeline());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPipelineBreakerBroadcastedPartialSolution() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
-
- DataSet<Long> initialSource = env.generateSequence(1, 10);
- IterativeDataSet<Long> iteration = initialSource.iterate(100);
-
-
- DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
-
- DataSet<Long> bcInput1 = sourceWithMapper
- .map(new IdentityMapper<Long>())
- .reduce(new SelectOneReducer<Long>());
-
- DataSet<Long> result = sourceWithMapper
- .map(new IdentityMapper<Long>())
- .withBroadcastSet(iteration, "bc2")
- .withBroadcastSet(bcInput1, "bc1");
-
-
- iteration.closeWith(result).print();
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- BulkIterationPlanNode iterationPlanNode = (BulkIterationPlanNode) sink.getInput().getSource();
- SingleInputPlanNode mapper = (SingleInputPlanNode) iterationPlanNode.getRootOfStepFunction();
-
- assertTrue(mapper.getInput().getTempMode().breaksPipeline());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testPilelineBreakerWithCross() {
- try {
- {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> initialSource = env.generateSequence(1, 10);
-
- Configuration conf= new Configuration();
- conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST);
- initialSource
- .map(new IdentityMapper<Long>())
- .cross(initialSource).withParameters(conf)
- .print();
-
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
- }
-
- {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> initialSource = env.generateSequence(1, 10);
-
- Configuration conf= new Configuration();
- conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
- initialSource
- .map(new IdentityMapper<Long>())
- .cross(initialSource).withParameters(conf)
- .print();
-
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
- }
-
- {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> initialSource = env.generateSequence(1, 10);
-
- Configuration conf= new Configuration();
- conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
- initialSource
- .map(new IdentityMapper<Long>())
- .cross(initialSource).withParameters(conf)
- .print();
-
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
- }
-
- {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(64);
-
- DataSet<Long> initialSource = env.generateSequence(1, 10);
-
- Configuration conf= new Configuration();
- conf.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
- initialSource
- .map(new IdentityMapper<Long>())
- .cross(initialSource).withParameters(conf)
- .print();
-
-
- Plan p = env.createProgramPlan();
- OptimizedPlan op = compileNoStats(p);
-
- SinkPlanNode sink = op.getDataSinks().iterator().next();
- DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
-
- assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
deleted file mode 100644
index 7be2b16..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/PropertyDataSourceTest.java
+++ /dev/null
@@ -1,897 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.DataSource;
-import org.apache.flink.api.java.operators.translation.JavaPlan;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@SuppressWarnings({"serial"})
-public class PropertyDataSourceTest extends CompilerTestBase {
-
- private List<Tuple3<Long, SomePojo, String>> tuple3PojoData = new ArrayList<Tuple3<Long, SomePojo, String>>();
- private TupleTypeInfo<Tuple3<Long, SomePojo, String>> tuple3PojoType = new TupleTypeInfo<Tuple3<Long, SomePojo, String>>(
- BasicTypeInfo.LONG_TYPE_INFO,
- TypeExtractor.createTypeInfo(SomePojo.class),
- BasicTypeInfo.STRING_TYPE_INFO
- );
-
- @Test
- public void checkSinglePartitionedSource1() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource2() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(1, 0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource3() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("*");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1, 2, 3, 4)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource4() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1, 2, 3)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource5() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.stringField");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(3)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource6() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.intField; f2");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2, 4)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedSource7() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("byDate", 1, 0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
- Assert.assertTrue(gprops.getCustomPartitioner() != null);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedGroupedSource1() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0)
- .splitsGroupedBy(0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedGroupedSource2() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0)
- .splitsGroupedBy(1, 0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0, 1)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedGroupedSource3() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(1)
- .splitsGroupedBy(0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedGroupedSource4() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0, 1)
- .splitsGroupedBy(0);
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedGroupedSource5() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f2")
- .splitsGroupedBy("f2");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(4)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(4)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedGroupedSource6() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.intField")
- .splitsGroupedBy("f0; f1.intField");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0,2)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedGroupedSource7() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.intField")
- .splitsGroupedBy("f1");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedGroupedSource8() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1")
- .splitsGroupedBy("f1.stringField");
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedOrderedSource1() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(1)
- .splitsOrderedBy(new int[]{1}, new Order[]{Order.ASCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedOrderedSource2() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(1)
- .splitsOrderedBy(new int[]{1, 0}, new Order[]{Order.ASCENDING, Order.DESCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1, 0)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedOrderedSource3() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0)
- .splitsOrderedBy(new int[]{1}, new Order[]{Order.ASCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedOrderedSource4() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy(0, 1)
- .splitsOrderedBy(new int[]{1}, new Order[]{Order.DESCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedOrderedSource5() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.intField")
- .splitsOrderedBy("f0; f1.intField", new Order[]{Order.ASCENDING, Order.DESCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0,2)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkSinglePartitionedOrderedSource6() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1.intField")
- .splitsOrderedBy("f1", new Order[]{Order.DESCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3)));
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
- @Test
- public void checkSinglePartitionedOrderedSource7() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);
-
- data.getSplitDataProperties()
- .splitsPartitionedBy("f1")
- .splitsOrderedBy("f1.stringField", new Order[]{Order.ASCENDING});
-
- data.print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();
-
- GlobalProperties gprops = sourceNode.getGlobalProperties();
- LocalProperties lprops = sourceNode.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
- Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
- Assert.assertTrue(lprops.getGroupedFields() == null);
- Assert.assertTrue(lprops.getOrdering() == null);
-
- }
-
-
- @Test
- public void checkCoPartitionedSources1() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data1 =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data1.getSplitDataProperties()
- .splitsPartitionedBy("byDate", 0);
-
- DataSource<Tuple2<Long, String>> data2 =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data2.getSplitDataProperties()
- .splitsPartitionedBy("byDate", 0);
-
- data1.union(data2).print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
- SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();
-
- GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
- LocalProperties lprops1 = sourceNode1.getLocalProperties();
- GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
- LocalProperties lprops2 = sourceNode2.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
- Assert.assertTrue(lprops1.getGroupedFields() == null);
- Assert.assertTrue(lprops1.getOrdering() == null);
-
- Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
- Assert.assertTrue(lprops2.getGroupedFields() == null);
- Assert.assertTrue(lprops2.getOrdering() == null);
-
- Assert.assertTrue(gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
- }
-
- @Test
- public void checkCoPartitionedSources2() {
-
- ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
-
- DataSource<Tuple2<Long, String>> data1 =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data1.getSplitDataProperties()
- .splitsPartitionedBy("byCountry", 0);
-
- DataSource<Tuple2<Long, String>> data2 =
- env.readCsvFile("/some/path").types(Long.class, String.class);
-
- data2.getSplitDataProperties()
- .splitsPartitionedBy("byDate", 0);
-
- data1.union(data2).print();
-
- JavaPlan plan = env.createProgramPlan();
-
- // submit the plan to the compiler
- OptimizedPlan oPlan = compileNoStats(plan);
-
- // check the optimized Plan
- SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
- SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
- SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();
-
- GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
- LocalProperties lprops1 = sourceNode1.getLocalProperties();
- GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
- LocalProperties lprops2 = sourceNode2.getLocalProperties();
-
- Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
- Assert.assertTrue(lprops1.getGroupedFields() == null);
- Assert.assertTrue(lprops1.getOrdering() == null);
-
- Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
- Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
- Assert.assertTrue(lprops2.getGroupedFields() == null);
- Assert.assertTrue(lprops2.getOrdering() == null);
-
- Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
- }
-
-
- public static class SomePojo {
- public double doubleField;
- public int intField;
- public String stringField;
- }
-
-}
-
-
[33/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
deleted file mode 100644
index 7ae35c3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-public class NoOpDescriptor extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.UNARY_NO_OP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "Pipe", in, DriverStrategy.UNARY_NO_OP);
- }
-
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return gProps;
- }
-
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
deleted file mode 100644
index c21593e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-
-/**
- *
- */
-public abstract class OperatorDescriptorDual implements AbstractOperatorDescriptor {
-
- protected final FieldList keys1;
- protected final FieldList keys2;
-
- private List<GlobalPropertiesPair> globalProps;
- private List<LocalPropertiesPair> localProps;
-
- protected OperatorDescriptorDual() {
- this(null, null);
- }
-
- protected OperatorDescriptorDual(FieldList keys1, FieldList keys2) {
- this.keys1 = keys1;
- this.keys2 = keys2;
- }
-
- public List<GlobalPropertiesPair> getPossibleGlobalProperties() {
- if (this.globalProps == null) {
- this.globalProps = createPossibleGlobalProperties();
- }
-
- return this.globalProps;
- }
-
- public List<LocalPropertiesPair> getPossibleLocalProperties() {
- if (this.localProps == null) {
- this.localProps = createPossibleLocalProperties();
- }
-
- return this.localProps;
- }
-
- protected abstract List<GlobalPropertiesPair> createPossibleGlobalProperties();
-
- protected abstract List<LocalPropertiesPair> createPossibleLocalProperties();
-
- public abstract boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2);
-
- public abstract boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2);
-
- public abstract DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node);
-
- public abstract GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2);
-
- public abstract LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2);
-
- protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2) {
-
- // check number of produced partitioning fields
- if(fields1.size() != fields2.size()) {
- return false;
- } else {
- return checkEquivalentFieldPositionsInKeyFields(fields1, fields2, fields1.size());
- }
- }
-
- protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2, int numRelevantFields) {
-
- // check number of produced partitioning fields
- if(fields1.size() < numRelevantFields || fields2.size() < numRelevantFields) {
- return false;
- }
- else {
- for(int i=0; i<numRelevantFields; i++) {
- int pField1 = fields1.get(i);
- int pField2 = fields2.get(i);
- // check if position of both produced fields is the same in both requested fields
- int j;
- for(j=0; j<this.keys1.size(); j++) {
- if(this.keys1.get(j) == pField1 && this.keys2.get(j) == pField2) {
- break;
- }
- else if(this.keys1.get(j) != pField1 && this.keys2.get(j) != pField2) {
- // do nothing
- }
- else {
- return false;
- }
- }
- if(j == this.keys1.size()) {
- throw new CompilerException("Fields were not found in key fields.");
- }
- }
- }
- return true;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static final class GlobalPropertiesPair {
-
- private final RequestedGlobalProperties props1, props2;
-
- public GlobalPropertiesPair(RequestedGlobalProperties props1, RequestedGlobalProperties props2) {
- this.props1 = props1;
- this.props2 = props2;
- }
-
- public RequestedGlobalProperties getProperties1() {
- return this.props1;
- }
-
- public RequestedGlobalProperties getProperties2() {
- return this.props2;
- }
-
- @Override
- public int hashCode() {
- return (this.props1 == null ? 0 : this.props1.hashCode()) ^ (this.props2 == null ? 0 : this.props2.hashCode());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == GlobalPropertiesPair.class) {
- final GlobalPropertiesPair other = (GlobalPropertiesPair) obj;
-
- return (this.props1 == null ? other.props1 == null : this.props1.equals(other.props1)) &&
- (this.props2 == null ? other.props2 == null : this.props2.equals(other.props2));
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "{" + this.props1 + " / " + this.props2 + "}";
- }
- }
-
- public static final class LocalPropertiesPair {
-
- private final RequestedLocalProperties props1, props2;
-
- public LocalPropertiesPair(RequestedLocalProperties props1, RequestedLocalProperties props2) {
- this.props1 = props1;
- this.props2 = props2;
- }
-
- public RequestedLocalProperties getProperties1() {
- return this.props1;
- }
-
- public RequestedLocalProperties getProperties2() {
- return this.props2;
- }
-
- @Override
- public int hashCode() {
- return (this.props1 == null ? 0 : this.props1.hashCode()) ^ (this.props2 == null ? 0 : this.props2.hashCode());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj.getClass() == LocalPropertiesPair.class) {
- final LocalPropertiesPair other = (LocalPropertiesPair) obj;
-
- return (this.props1 == null ? other.props1 == null : this.props1.equals(other.props1)) &&
- (this.props2 == null ? other.props2 == null : this.props2.equals(other.props2));
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "{" + this.props1 + " / " + this.props2 + "}";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
deleted file mode 100644
index c8be5d4..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-
-/**
- * Abstract base class for Operator descriptions which instantiates the node and sets the driver
- * strategy and the sorting and grouping keys. Returns possible local and global properties and
- * updates them after the operation has been performed.
- * @see org.apache.flink.compiler.dag.SingleInputNode
- */
-public abstract class OperatorDescriptorSingle implements AbstractOperatorDescriptor {
-
- protected final FieldSet keys; // the set of key fields
- protected final FieldList keyList; // the key fields with ordered field positions
-
- private List<RequestedGlobalProperties> globalProps;
- private List<RequestedLocalProperties> localProps;
-
-
- protected OperatorDescriptorSingle() {
- this(null);
- }
-
- protected OperatorDescriptorSingle(FieldSet keys) {
- this.keys = keys;
- this.keyList = keys == null ? null : keys.toFieldList();
- }
-
-
- public List<RequestedGlobalProperties> getPossibleGlobalProperties() {
- if (this.globalProps == null) {
- this.globalProps = createPossibleGlobalProperties();
- }
- return this.globalProps;
- }
-
- public List<RequestedLocalProperties> getPossibleLocalProperties() {
- if (this.localProps == null) {
- this.localProps = createPossibleLocalProperties();
- }
- return this.localProps;
- }
-
- /**
- * Returns a list of global properties that are required by this operator descriptor.
- *
- * @return A list of global properties that are required by this operator descriptor.
- */
- protected abstract List<RequestedGlobalProperties> createPossibleGlobalProperties();
-
- /**
- * Returns a list of local properties that are required by this operator descriptor.
- *
- * @return A list of local properties that are required by this operator descriptor.
- */
- protected abstract List<RequestedLocalProperties> createPossibleLocalProperties();
-
- public abstract SingleInputPlanNode instantiate(Channel in, SingleInputNode node);
-
- /**
- * Returns the global properties which are present after the operator was applied on the
- * provided global properties.
- *
- * @param in The global properties on which the operator is applied.
- * @return The global properties which are valid after the operator has been applied.
- */
- public abstract GlobalProperties computeGlobalProperties(GlobalProperties in);
-
- /**
- * Returns the local properties which are present after the operator was applied on the
- * provided local properties.
- *
- * @param in The local properties on which the operator is applied.
- * @return The local properties which are valid after the operator has been applied.
- */
- public abstract LocalProperties computeLocalProperties(LocalProperties in);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
deleted file mode 100644
index 2bde29b..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-public final class PartialGroupProperties extends OperatorDescriptorSingle {
-
- public PartialGroupProperties(FieldSet keys) {
- super(keys);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.SORTED_GROUP_COMBINE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- // create in input node for combine with same DOP as input node
- GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
- combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
-
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator().getName()+")", in,
- DriverStrategy.SORTED_GROUP_COMBINE);
- // sorting key info
- combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
- // set grouping comparator key info
- combiner.setDriverKeyInfo(this.keyList, 1);
-
- return combiner;
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- RequestedLocalProperties props = new RequestedLocalProperties();
- props.setGroupedFields(this.keys);
- return Collections.singletonList(props);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
deleted file mode 100644
index 5bb51f3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.ReduceNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-
-public final class ReduceProperties extends OperatorDescriptorSingle {
-
- private final Partitioner<?> customPartitioner;
-
- public ReduceProperties(FieldSet keys) {
- this(keys, null);
- }
-
- public ReduceProperties(FieldSet keys, Partitioner<?> customPartitioner) {
- super(keys);
- this.customPartitioner = customPartitioner;
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.SORTED_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
- (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty()))
- {
- return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
- DriverStrategy.SORTED_REDUCE, this.keyList);
- }
- else {
- // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
- Channel toCombiner = new Channel(in.getSource());
- toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
-
- // create an input node for combine with same DOP as input node
- ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
- combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
-
- SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
- "Combine ("+node.getOperator().getName()+")", toCombiner,
- DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList);
-
- combiner.setCosts(new Costs(0, 0));
- combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
-
- Channel toReducer = new Channel(combiner);
- toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
- in.getShipStrategySortOrder(), in.getDataExchangeMode());
- toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
-
- return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", toReducer,
- DriverStrategy.SORTED_REDUCE, this.keyList);
- }
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties props = new RequestedGlobalProperties();
- if (customPartitioner == null) {
- props.setAnyPartitioning(this.keys);
- } else {
- props.setCustomPartitioned(this.keys, this.customPartitioner);
- }
- return Collections.singletonList(props);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- RequestedLocalProperties props = new RequestedLocalProperties();
- props.setGroupedFields(this.keys);
- return Collections.singletonList(props);
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
deleted file mode 100644
index 1dcd87d..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class SolutionSetDeltaOperator extends OperatorDescriptorSingle {
-
- public SolutionSetDeltaOperator(FieldList partitioningFields) {
- super(partitioningFields);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.UNARY_NO_OP;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "SolutionSet Delta", in, DriverStrategy.UNARY_NO_OP);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- RequestedGlobalProperties partProps = new RequestedGlobalProperties();
- partProps.setHashPartitioned(this.keyList);
- return Collections.singletonList(partProps);
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- return gProps;
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
deleted file mode 100644
index 356836a..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class SortMergeJoinDescriptor extends AbstractJoinDescriptor {
-
- public SortMergeJoinDescriptor(FieldList keys1, FieldList keys2) {
- super(keys1, keys2);
- }
-
- public SortMergeJoinDescriptor(FieldList keys1, FieldList keys2,
- boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
- {
- super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
- }
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.MERGE;
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- RequestedLocalProperties sort1 = new RequestedLocalProperties(Utils.createOrdering(this.keys1));
- RequestedLocalProperties sort2 = new RequestedLocalProperties(Utils.createOrdering(this.keys2));
- return Collections.singletonList(new LocalPropertiesPair(sort1, sort2));
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2)
- {
- int numRelevantFields = this.keys1.size();
-
- Ordering prod1 = produced1.getOrdering();
- Ordering prod2 = produced2.getOrdering();
-
- if (prod1 == null || prod2 == null) {
- throw new CompilerException("The given properties do not meet this operators requirements.");
- }
-
- // check that order of fields is equivalent
- if (!checkEquivalentFieldPositionsInKeyFields(
- prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
- return false;
- }
-
- // check that both inputs have the same directions of order
- for (int i = 0; i < numRelevantFields; i++) {
- if (prod1.getOrder(i) != prod2.getOrder(i)) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- boolean[] inputOrders = in1.getLocalProperties().getOrdering().getFieldSortDirections();
-
- if (inputOrders == null || inputOrders.length < this.keys1.size()) {
- throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a merge operator.");
- } else if (inputOrders.length > this.keys1.size()) {
- boolean[] tmp = new boolean[this.keys1.size()];
- System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
- inputOrders = tmp;
- }
-
- return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, DriverStrategy.MERGE, this.keys1, this.keys2, inputOrders);
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- LocalProperties comb = LocalProperties.combine(in1, in2);
- return comb.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
deleted file mode 100644
index c42cff2..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- *
- */
-public class UtilSinkJoinOpDescriptor extends OperatorDescriptorDual {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.BINARY_NO_OP;
- }
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
- // all properties are possible
- return Collections.singletonList(new GlobalPropertiesPair(
- new RequestedGlobalProperties(), new RequestedGlobalProperties()));
- }
-
- @Override
- protected List<LocalPropertiesPair> createPossibleLocalProperties() {
- // all properties are possible
- return Collections.singletonList(new LocalPropertiesPair(
- new RequestedLocalProperties(), new RequestedLocalProperties()));
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2) {
- return true;
- }
-
- @Override
- public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
- LocalProperties produced1, LocalProperties produced2) {
- return true;
- }
-
- @Override
- public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
- if (node instanceof SinkJoiner) {
- return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2);
- } else {
- throw new CompilerException();
- }
- }
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
- return new LocalProperties();
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- return GlobalProperties.combine(in1, in2);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
deleted file mode 100644
index bf22fb3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- * A special subclass for the union to make it identifiable.
- */
-public class BinaryUnionPlanNode extends DualInputPlanNode {
-
- /**
- * @param template
- */
- public BinaryUnionPlanNode(BinaryUnionNode template, Channel in1, Channel in2) {
- super(template, "Union", in1, in2, DriverStrategy.UNION);
- }
-
- public BinaryUnionPlanNode(BinaryUnionPlanNode toSwapFrom) {
- super(toSwapFrom.getOptimizerNode(), "Union-With-Cached", toSwapFrom.getInput2(), toSwapFrom.getInput1(),
- DriverStrategy.UNION_WITH_CACHED);
-
- this.globalProps = toSwapFrom.globalProps;
- this.localProps = toSwapFrom.localProps;
- this.nodeCosts = toSwapFrom.nodeCosts;
- this.cumulativeCosts = toSwapFrom.cumulativeCosts;
-
- setParallelism(toSwapFrom.getParallelism());
- }
-
- public BinaryUnionNode getOptimizerNode() {
- return (BinaryUnionNode) this.template;
- }
-
- public boolean unionsStaticAndDynamicPath() {
- return getInput1().isOnDynamicPath() != getInput2().isOnDynamicPath();
- }
-
- @Override
- public int getMemoryConsumerWeight() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
deleted file mode 100644
index e79e2f3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-
-import java.util.HashMap;
-
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.BulkIterationNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-public class BulkIterationPlanNode extends SingleInputPlanNode implements IterationPlanNode {
-
- private final BulkPartialSolutionPlanNode partialSolutionPlanNode;
-
- private final PlanNode rootOfStepFunction;
-
- private PlanNode rootOfTerminationCriterion;
-
- private TypeSerializerFactory<?> serializerForIterationChannel;
-
- // --------------------------------------------------------------------------------------------
-
- public BulkIterationPlanNode(BulkIterationNode template, String nodeName, Channel input,
- BulkPartialSolutionPlanNode pspn, PlanNode rootOfStepFunction)
- {
- super(template, nodeName, input, DriverStrategy.NONE);
- this.partialSolutionPlanNode = pspn;
- this.rootOfStepFunction = rootOfStepFunction;
-
- mergeBranchPlanMaps();
- }
-
- public BulkIterationPlanNode(BulkIterationNode template, String nodeName, Channel input,
- BulkPartialSolutionPlanNode pspn, PlanNode rootOfStepFunction, PlanNode rootOfTerminationCriterion)
- {
- this(template, nodeName, input, pspn, rootOfStepFunction);
- this.rootOfTerminationCriterion = rootOfTerminationCriterion;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public BulkIterationNode getIterationNode() {
- if (this.template instanceof BulkIterationNode) {
- return (BulkIterationNode) this.template;
- } else {
- throw new RuntimeException();
- }
- }
-
- public BulkPartialSolutionPlanNode getPartialSolutionPlanNode() {
- return this.partialSolutionPlanNode;
- }
-
- public PlanNode getRootOfStepFunction() {
- return this.rootOfStepFunction;
- }
-
- public PlanNode getRootOfTerminationCriterion() {
- return this.rootOfTerminationCriterion;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- public TypeSerializerFactory<?> getSerializerForIterationChannel() {
- return serializerForIterationChannel;
- }
-
- public void setSerializerForIterationChannel(TypeSerializerFactory<?> serializerForIterationChannel) {
- this.serializerForIterationChannel = serializerForIterationChannel;
- }
-
- public void setCosts(Costs nodeCosts) {
- // add the costs from the step function
- nodeCosts.addCosts(this.rootOfStepFunction.getCumulativeCosts());
-
- // add the costs for the termination criterion, if it exists
- // the costs are divided at branches, so we can simply add them up
- if (rootOfTerminationCriterion != null) {
- nodeCosts.addCosts(this.rootOfTerminationCriterion.getCumulativeCosts());
- }
-
- super.setCosts(nodeCosts);
- }
-
- public int getMemoryConsumerWeight() {
- return 1;
- }
-
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
-
- SourceAndDamReport fromOutside = super.hasDamOnPathDownTo(source);
-
- if (fromOutside == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (fromOutside == FOUND_SOURCE) {
- // we always have a dam in the back channel
- return FOUND_SOURCE_AND_DAM;
- } else {
- // check the step function for dams
- return this.rootOfStepFunction.hasDamOnPathDownTo(source);
- }
- }
-
- @Override
- public void acceptForStepFunction(Visitor<PlanNode> visitor) {
- this.rootOfStepFunction.accept(visitor);
-
- if(this.rootOfTerminationCriterion != null) {
- this.rootOfTerminationCriterion.accept(visitor);
- }
- }
-
- private void mergeBranchPlanMaps() {
- for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) {
- OptimizerNode brancher = desc.getBranchingNode();
-
- if (branchPlan == null) {
- branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
- }
-
- if (!branchPlan.containsKey(brancher)) {
- PlanNode selectedCandidate = null;
-
- if (rootOfStepFunction.branchPlan != null) {
- selectedCandidate = rootOfStepFunction.branchPlan.get(brancher);
- }
-
- if (selectedCandidate == null) {
- throw new CompilerException(
- "Candidates for a node with open branches are missing information about the selected candidate ");
- }
-
- this.branchPlan.put(brancher, selectedCandidate);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
deleted file mode 100644
index df05b64..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.Collections;
-import java.util.HashMap;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * Plan candidate node for partial solution of a bulk iteration.
- */
-public class BulkPartialSolutionPlanNode extends PlanNode {
-
- private static final Costs NO_COSTS = new Costs();
-
- private BulkIterationPlanNode containingIterationNode;
-
- private Channel initialInput;
-
- public Object postPassHelper;
-
-
- public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
- GlobalProperties gProps, LocalProperties lProps,
- Channel initialInput)
- {
- super(template, nodeName, DriverStrategy.NONE);
-
- this.globalProps = gProps;
- this.localProps = lProps;
- this.initialInput = initialInput;
-
- // the partial solution does not cost anything
- this.nodeCosts = NO_COSTS;
- this.cumulativeCosts = NO_COSTS;
-
- if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
- }
-
- this.branchPlan.putAll(initialInput.getSource().branchPlan);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public BulkPartialSolutionNode getPartialSolutionNode() {
- return (BulkPartialSolutionNode) this.template;
- }
-
- public BulkIterationPlanNode getContainingIterationNode() {
- return this.containingIterationNode;
- }
-
- public void setContainingIterationNode(BulkIterationPlanNode containingIterationNode) {
- this.containingIterationNode = containingIterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- return Collections.<PlanNode>emptyList();
- }
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.<Channel>emptyList();
- }
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
- SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
- if (res == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (res == FOUND_SOURCE) {
- return (this.initialInput.getLocalStrategy().dams() ||
- this.initialInput.getTempMode().breaksPipeline() ||
- getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
- FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
- }
- else {
- return NOT_FOUND;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
deleted file mode 100644
index 875d1c3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/Channel.java
+++ /dev/null
@@ -1,538 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.EstimateProvider;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plandump.DumpableConnection;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-
-/**
- * A Channel represents the result produced by an operator and the data exchange
- * before the consumption by the target operator.
- *
- * The channel defines and tracks various properties and characteristics of the
- * data set and data exchange.
- *
- * Data set characteristics:
- * <ul>
- * <li>The "global properties" of the data, i.e., how the data is distributed across
- * partitions</li>
- * <li>The "required global properties" of the data, i.e., the global properties that, if absent,
- * would cause the program to return a wrong result.</li>
- * <li>The "local properties" of the data, i.e., how the data is organized within a partition</li>
- * <li>The "required local properties" of the data, i.e., the local properties that, if absent,
- * would cause the program to return a wrong result.</li>
- * </ul>
- *
- * Data exchange parameters:
- * <ul>
- * <li>The "ship strategy", i.e., whether to forward the data, shuffle it, broadcast it, ...</li>
- * <li>The "ship keys", which are the positions of the key fields in the exchanged records.</li>
- * <li>The "data exchange mode", which defines whether to pipeline or batch the exchange</li>
- * <li>Several more...</li>
- * </ul>
- */
-public class Channel implements EstimateProvider, Cloneable, DumpableConnection<PlanNode> {
-
- private PlanNode source;
-
- private PlanNode target;
-
- private ShipStrategyType shipStrategy = ShipStrategyType.NONE;
-
- private DataExchangeMode dataExchangeMode;
-
- private LocalStrategy localStrategy = LocalStrategy.NONE;
-
- private FieldList shipKeys;
-
- private FieldList localKeys;
-
- private boolean[] shipSortOrder;
-
- private boolean[] localSortOrder;
-
- private RequestedGlobalProperties requiredGlobalProps;
-
- private RequestedLocalProperties requiredLocalProps;
-
- private GlobalProperties globalProps;
-
- private LocalProperties localProps;
-
- private TypeSerializerFactory<?> serializer;
-
- private TypeComparatorFactory<?> shipStrategyComparator;
-
- private TypeComparatorFactory<?> localStrategyComparator;
-
- private DataDistribution dataDistribution;
-
- private Partitioner<?> partitioner;
-
- private TempMode tempMode;
-
- private double relativeTempMemory;
-
- private double relativeMemoryLocalStrategy;
-
- private int replicationFactor = 1;
-
- // --------------------------------------------------------------------------------------------
-
- public Channel(PlanNode sourceNode) {
- this(sourceNode, null);
- }
-
- public Channel(PlanNode sourceNode, TempMode tempMode) {
- this.source = sourceNode;
- this.tempMode = (tempMode == null ? TempMode.NONE : tempMode);
- }
-
- // --------------------------------------------------------------------------------------------
- // Accessors
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the source of this Channel.
- *
- * @return The source.
- */
- @Override
- public PlanNode getSource() {
- return this.source;
- }
-
- /**
- * Sets the target of this Channel.
- *
- * @param target The target.
- */
- public void setTarget(PlanNode target) {
- this.target = target;
- }
-
- /**
- * Gets the target of this Channel.
- *
- * @return The target.
- */
- public PlanNode getTarget() {
- return this.target;
- }
-
- public void setShipStrategy(ShipStrategyType strategy, DataExchangeMode dataExchangeMode) {
- setShipStrategy(strategy, null, null, null, dataExchangeMode);
- }
-
- public void setShipStrategy(ShipStrategyType strategy, FieldList keys, DataExchangeMode dataExchangeMode) {
- setShipStrategy(strategy, keys, null, null, dataExchangeMode);
- }
-
- public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
- boolean[] sortDirection, DataExchangeMode dataExchangeMode) {
- setShipStrategy(strategy, keys, sortDirection, null, dataExchangeMode);
- }
-
- public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
- Partitioner<?> partitioner, DataExchangeMode dataExchangeMode) {
- setShipStrategy(strategy, keys, null, partitioner, dataExchangeMode);
- }
-
- public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
- boolean[] sortDirection, Partitioner<?> partitioner,
- DataExchangeMode dataExchangeMode) {
- this.shipStrategy = strategy;
- this.shipKeys = keys;
- this.shipSortOrder = sortDirection;
- this.partitioner = partitioner;
- this.dataExchangeMode = dataExchangeMode;
- this.globalProps = null; // reset the global properties
- }
-
- /**
- * Gets the data exchange mode (batch / streaming) to use for the data
- * exchange of this channel.
- *
- * @return The data exchange mode of this channel.
- */
- public DataExchangeMode getDataExchangeMode() {
- return dataExchangeMode;
- }
-
- public ShipStrategyType getShipStrategy() {
- return this.shipStrategy;
- }
-
- public FieldList getShipStrategyKeys() {
- return this.shipKeys;
- }
-
- public boolean[] getShipStrategySortOrder() {
- return this.shipSortOrder;
- }
-
- public void setLocalStrategy(LocalStrategy strategy) {
- setLocalStrategy(strategy, null, null);
- }
-
- public void setLocalStrategy(LocalStrategy strategy, FieldList keys, boolean[] sortDirection) {
- this.localStrategy = strategy;
- this.localKeys = keys;
- this.localSortOrder = sortDirection;
- this.localProps = null; // reset the local properties
- }
-
- public LocalStrategy getLocalStrategy() {
- return this.localStrategy;
- }
-
- public FieldList getLocalStrategyKeys() {
- return this.localKeys;
- }
-
- public boolean[] getLocalStrategySortOrder() {
- return this.localSortOrder;
- }
-
- public void setDataDistribution(DataDistribution dataDistribution) {
- this.dataDistribution = dataDistribution;
- }
-
- public DataDistribution getDataDistribution() {
- return this.dataDistribution;
- }
-
- public Partitioner<?> getPartitioner() {
- return partitioner;
- }
-
- public TempMode getTempMode() {
- return this.tempMode;
- }
-
- /**
- * Sets the temp mode of the connection.
- *
- * @param tempMode
- * The temp mode of the connection.
- */
- public void setTempMode(TempMode tempMode) {
- this.tempMode = tempMode;
- }
-
- /**
- * Gets the memory for materializing the channel's result from this Channel.
- *
- * @return The temp memory.
- */
- public double getRelativeTempMemory() {
- return this.relativeTempMemory;
- }
-
- /**
- * Sets the memory for materializing the channel's result from this Channel.
- *
- * @param relativeTempMemory The memory for materialization.
- */
- public void setRelativeTempMemory(double relativeTempMemory) {
- this.relativeTempMemory = relativeTempMemory;
- }
-
- /**
- * Sets the replication factor of the connection.
- *
- * @param factor The replication factor of the connection.
- */
- public void setReplicationFactor(int factor) {
- this.replicationFactor = factor;
- }
-
- /**
- * Returns the replication factor of the connection.
- *
- * @return The replication factor of the connection.
- */
- public int getReplicationFactor() {
- return this.replicationFactor;
- }
-
- /**
- * Gets the serializer from this Channel.
- *
- * @return The serializer.
- */
- public TypeSerializerFactory<?> getSerializer() {
- return serializer;
- }
-
- /**
- * Sets the serializer for this Channel.
- *
- * @param serializer The serializer to set.
- */
- public void setSerializer(TypeSerializerFactory<?> serializer) {
- this.serializer = serializer;
- }
-
- /**
- * Gets the ship strategy comparator from this Channel.
- *
- * @return The ship strategy comparator.
- */
- public TypeComparatorFactory<?> getShipStrategyComparator() {
- return shipStrategyComparator;
- }
-
- /**
- * Sets the ship strategy comparator for this Channel.
- *
- * @param shipStrategyComparator The ship strategy comparator to set.
- */
- public void setShipStrategyComparator(TypeComparatorFactory<?> shipStrategyComparator) {
- this.shipStrategyComparator = shipStrategyComparator;
- }
-
- /**
- * Gets the local strategy comparator from this Channel.
- *
- * @return The local strategy comparator.
- */
- public TypeComparatorFactory<?> getLocalStrategyComparator() {
- return localStrategyComparator;
- }
-
- /**
- * Sets the local strategy comparator for this Channel.
- *
- * @param localStrategyComparator The local strategy comparator to set.
- */
- public void setLocalStrategyComparator(TypeComparatorFactory<?> localStrategyComparator) {
- this.localStrategyComparator = localStrategyComparator;
- }
-
- public double getRelativeMemoryLocalStrategy() {
- return relativeMemoryLocalStrategy;
- }
-
- public void setRelativeMemoryLocalStrategy(double relativeMemoryLocalStrategy) {
- this.relativeMemoryLocalStrategy = relativeMemoryLocalStrategy;
- }
-
- public boolean isOnDynamicPath() {
- return this.source.isOnDynamicPath();
- }
-
- public int getCostWeight() {
- return this.source.getCostWeight();
- }
-
- // --------------------------------------------------------------------------------------------
- // Statistic Estimates
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public long getEstimatedOutputSize() {
- long estimate = this.source.template.getEstimatedOutputSize();
- return estimate < 0 ? estimate : estimate * this.replicationFactor;
- }
-
- @Override
- public long getEstimatedNumRecords() {
- long estimate = this.source.template.getEstimatedNumRecords();
- return estimate < 0 ? estimate : estimate * this.replicationFactor;
- }
-
- @Override
- public float getEstimatedAvgWidthPerOutputRecord() {
- return this.source.template.getEstimatedAvgWidthPerOutputRecord();
- }
-
- // --------------------------------------------------------------------------------------------
- // Data Property Handling
- // --------------------------------------------------------------------------------------------
-
-
- public RequestedGlobalProperties getRequiredGlobalProps() {
- return requiredGlobalProps;
- }
-
- public void setRequiredGlobalProps(RequestedGlobalProperties requiredGlobalProps) {
- this.requiredGlobalProps = requiredGlobalProps;
- }
-
- public RequestedLocalProperties getRequiredLocalProps() {
- return requiredLocalProps;
- }
-
- public void setRequiredLocalProps(RequestedLocalProperties requiredLocalProps) {
- this.requiredLocalProps = requiredLocalProps;
- }
-
- public GlobalProperties getGlobalProperties() {
- if (this.globalProps == null) {
- this.globalProps = this.source.getGlobalProperties().clone();
- switch (this.shipStrategy) {
- case BROADCAST:
- this.globalProps.clearUniqueFieldCombinations();
- this.globalProps.setFullyReplicated();
- break;
- case PARTITION_HASH:
- this.globalProps.setHashPartitioned(this.shipKeys);
- break;
- case PARTITION_RANGE:
- this.globalProps.setRangePartitioned(Utils.createOrdering(this.shipKeys, this.shipSortOrder));
- break;
- case FORWARD:
- break;
- case PARTITION_RANDOM:
- this.globalProps.reset();
- break;
- case PARTITION_FORCED_REBALANCE:
- this.globalProps.setForcedRebalanced();
- break;
- case PARTITION_CUSTOM:
- this.globalProps.setCustomPartitioned(this.shipKeys, this.partitioner);
- break;
- case NONE:
- throw new CompilerException("Cannot produce GlobalProperties before ship strategy is set.");
- }
- }
-
- return this.globalProps;
- }
-
- public LocalProperties getLocalProperties() {
- if (this.localProps == null) {
- computeLocalPropertiesAfterShippingOnly();
- switch (this.localStrategy) {
- case NONE:
- break;
- case SORT:
- case COMBININGSORT:
- this.localProps = LocalProperties.forOrdering(Utils.createOrdering(this.localKeys, this.localSortOrder));
- break;
- default:
- throw new CompilerException("Unsupported local strategy for channel.");
- }
- }
-
- return this.localProps;
- }
-
- private void computeLocalPropertiesAfterShippingOnly() {
- switch (this.shipStrategy) {
- case BROADCAST:
- case PARTITION_HASH:
- case PARTITION_CUSTOM:
- case PARTITION_RANGE:
- case PARTITION_RANDOM:
- case PARTITION_FORCED_REBALANCE:
- this.localProps = new LocalProperties();
- break;
- case FORWARD:
- this.localProps = this.source.getLocalProperties();
- break;
- case NONE:
- throw new CompilerException("ShipStrategy has not yet been set.");
- default:
- throw new CompilerException("Unknown ShipStrategy.");
- }
- }
-
- public void adjustGlobalPropertiesForFullParallelismChange() {
- if (this.shipStrategy == null || this.shipStrategy == ShipStrategyType.NONE) {
- throw new IllegalStateException("Cannot adjust channel for degree of parallelism " +
- "change before the ship strategy is set.");
- }
-
- // make sure the properties are acquired
- if (this.globalProps == null) {
- getGlobalProperties();
- }
-
- // some strategies globally reestablish properties
- switch (this.shipStrategy) {
- case FORWARD:
- throw new CompilerException("Cannot use FORWARD strategy between operations " +
- "with different number of parallel instances.");
- case NONE: // excluded by sanity check. left here for verification check completion
- case BROADCAST:
- case PARTITION_HASH:
- case PARTITION_RANGE:
- case PARTITION_RANDOM:
- case PARTITION_FORCED_REBALANCE:
- case PARTITION_CUSTOM:
- return;
- }
- throw new CompilerException("Unrecognized Ship Strategy Type: " + this.shipStrategy);
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Utility method used while swapping binary union nodes for n-ary union nodes.
- */
- public void swapUnionNodes(PlanNode newUnionNode) {
- if (!(this.source instanceof BinaryUnionPlanNode)) {
- throw new IllegalStateException();
- } else {
- this.source = newUnionNode;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public int getMaxDepth() {
- return this.source.getOptimizerNode().getMaxDepth() + 1;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public String toString() {
- return "Channel (" + this.source + (this.target == null ? ')' : ") -> (" + this.target + ')') +
- '[' + this.shipStrategy + "] [" + this.localStrategy + "] " +
- (this.tempMode == null || this.tempMode == TempMode.NONE ? "{NO-TEMP}" : this.tempMode);
- }
-
- @Override
- public Channel clone() {
- try {
- return (Channel) super.clone();
- } catch (CloneNotSupportedException cnsex) {
- throw new RuntimeException(cnsex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
deleted file mode 100644
index 01c56dd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-
-/**
- *
- */
-public class DualInputPlanNode extends PlanNode {
-
- protected final Channel input1;
- protected final Channel input2;
-
- protected final FieldList keys1;
- protected final FieldList keys2;
-
- protected final boolean[] sortOrders;
-
- private TypeComparatorFactory<?> comparator1;
- private TypeComparatorFactory<?> comparator2;
- private TypePairComparatorFactory<?, ?> pairComparator;
-
- public Object postPassHelper1;
- public Object postPassHelper2;
-
- // --------------------------------------------------------------------------------------------
-
- public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy) {
- this(template, nodeName, input1, input2, diverStrategy, null, null, null);
- }
-
- public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2,
- DriverStrategy diverStrategy, FieldList driverKeyFields1, FieldList driverKeyFields2)
- {
- this(template, nodeName, input1, input2, diverStrategy, driverKeyFields1, driverKeyFields2,
- SingleInputPlanNode.getTrueArray(driverKeyFields1.size()));
- }
-
- public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy,
- FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders)
- {
- super(template, nodeName, diverStrategy);
- this.input1 = input1;
- this.input2 = input2;
- this.keys1 = driverKeyFields1;
- this.keys2 = driverKeyFields2;
- this.sortOrders = driverSortOrders;
-
- if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input1.setReplicationFactor(getParallelism());
- }
- if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input2.setReplicationFactor(getParallelism());
- }
-
- mergeBranchPlanMaps(input1.getSource(), input2.getSource());
- }
-
- // --------------------------------------------------------------------------------------------
-
- public TwoInputNode getTwoInputNode() {
- if (this.template instanceof TwoInputNode) {
- return (TwoInputNode) this.template;
- } else {
- throw new RuntimeException();
- }
- }
-
- public FieldList getKeysForInput1() {
- return this.keys1;
- }
-
- public FieldList getKeysForInput2() {
- return this.keys2;
- }
-
- public boolean[] getSortOrders() {
- return this.sortOrders;
- }
-
- public TypeComparatorFactory<?> getComparator1() {
- return this.comparator1;
- }
-
- public TypeComparatorFactory<?> getComparator2() {
- return this.comparator2;
- }
-
- public void setComparator1(TypeComparatorFactory<?> comparator) {
- this.comparator1 = comparator;
- }
-
- public void setComparator2(TypeComparatorFactory<?> comparator) {
- this.comparator2 = comparator;
- }
-
- public TypePairComparatorFactory<?, ?> getPairComparator() {
- return this.pairComparator;
- }
-
- public void setPairComparator(TypePairComparatorFactory<?, ?> comparator) {
- this.pairComparator = comparator;
- }
-
- /**
- * Gets the first input channel to this node.
- *
- * @return The first input channel to this node.
- */
- public Channel getInput1() {
- return this.input1;
- }
-
- /**
- * Gets the second input channel to this node.
- *
- * @return The second input channel to this node.
- */
- public Channel getInput2() {
- return this.input2;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- this.input1.getSource().accept(visitor);
- this.input2.getSource().accept(visitor);
-
- for (Channel broadcastInput : getBroadcastInputs()) {
- broadcastInput.getSource().accept(visitor);
- }
-
- visitor.postVisit(this);
- }
- }
-
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) {
- return Arrays.asList(this.input1.getSource(), this.input2.getSource());
- } else {
- List<PlanNode> preds = new ArrayList<PlanNode>();
-
- preds.add(input1.getSource());
- preds.add(input2.getSource());
-
- for (Channel c : getBroadcastInputs()) {
- preds.add(c.getSource());
- }
-
- return preds;
- }
- }
-
- @Override
- public Iterable<Channel> getInputs() {
- return Arrays.asList(this.input1, this.input2);
- }
-
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
-
- // check first input
- SourceAndDamReport res1 = this.input1.getSource().hasDamOnPathDownTo(source);
- if (res1 == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (res1 == FOUND_SOURCE) {
- if (this.input1.getLocalStrategy().dams() || this.input1.getTempMode().breaksPipeline() ||
- getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) {
- return FOUND_SOURCE_AND_DAM;
- } else {
- return FOUND_SOURCE;
- }
- }
- else {
- SourceAndDamReport res2 = this.input2.getSource().hasDamOnPathDownTo(source);
- if (res2 == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (res2 == FOUND_SOURCE) {
- if (this.input2.getLocalStrategy().dams() || this.input2.getTempMode().breaksPipeline() ||
- getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) {
- return FOUND_SOURCE_AND_DAM;
- } else {
- return FOUND_SOURCE;
- }
- }
- else {
- // NOT_FOUND
- // check the broadcast inputs
-
- for (NamedChannel nc : getBroadcastInputs()) {
- SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source);
- if (bcRes != NOT_FOUND) {
- // broadcast inputs are always dams
- return FOUND_SOURCE_AND_DAM;
- }
- }
- return NOT_FOUND;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
deleted file mode 100644
index d146c83..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-/**
- * A common interface for compiled Flink plans for both batch and streaming
- * processing programs.
- *
- */
-public interface FlinkPlan {
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
deleted file mode 100644
index 38f76b2..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.util.Visitor;
-
-/**
- *
- */
-public interface IterationPlanNode {
-
- void acceptForStepFunction(Visitor<PlanNode> visitor);
-
- IterationNode getIterationNode();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
deleted file mode 100644
index 3650eea..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.IterableIterator;
-import org.apache.flink.util.Visitor;
-
-/**
- * A union operation over multiple inputs (2 or more).
- */
-public class NAryUnionPlanNode extends PlanNode {
-
- private final List<Channel> inputs;
-
- /**
- * @param template
- */
- public NAryUnionPlanNode(BinaryUnionNode template, List<Channel> inputs, GlobalProperties gProps,
- Costs cumulativeCosts)
- {
- super(template, "Union", DriverStrategy.NONE);
-
- this.inputs = inputs;
- this.globalProps = gProps;
- this.localProps = new LocalProperties();
- this.nodeCosts = new Costs();
- this.cumulativeCosts = cumulativeCosts;
- }
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- visitor.preVisit(this);
- for (Channel c : this.inputs) {
- c.getSource().accept(visitor);
- }
- visitor.postVisit(this);
- }
-
- public List<Channel> getListOfInputs() {
- return this.inputs;
- }
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.unmodifiableList(this.inputs);
- }
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- final Iterator<Channel> channels = this.inputs.iterator();
- return new IterableIterator<PlanNode>() {
-
- @Override
- public boolean hasNext() {
- return channels.hasNext();
- }
-
- @Override
- public PlanNode next() {
- return channels.next().getSource();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Iterator<PlanNode> iterator() {
- return this;
- }
- };
- }
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- // this node is used after the plan enumeration. consequently, this will never be invoked here
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
deleted file mode 100644
index da97e61..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.optimizer.dag.TempMode;
-
-public class NamedChannel extends Channel {
-
- private final String name;
-
- /**
- * Initializes NamedChannel.
- *
- * @param sourceNode
- */
- public NamedChannel(String name, PlanNode sourceNode) {
- super(sourceNode);
- this.name = name;
- }
-
- public NamedChannel(String name, PlanNode sourceNode, TempMode tempMode) {
- super(sourceNode, tempMode);
- this.name = name;
- }
-
- public String getName() {
- return this.name;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
deleted file mode 100644
index d56be87..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import java.util.Collection;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.util.Visitable;
-import org.apache.flink.util.Visitor;
-
-/**
- * The execution plan generated by the Optimizer. It contains {@link PlanNode}s
- * and {@link Channel}s that describe exactly how the program should be executed.
- * It defines all ship strategies (local pipe, shuffle, broadcast, rebalance), all
- * operator strategies (sorting-merge join, hash join, sorted grouping, ...),
- * and the data exchange modes (batched, pipelined).
- */
-public class OptimizedPlan implements FlinkPlan, Visitable<PlanNode> {
-
- /** The data sources in the plan. */
- private final Collection<SourcePlanNode> dataSources;
-
- /** The data sinks in the plan. */
- private final Collection<SinkPlanNode> dataSinks;
-
- /** All nodes in the optimizer plan. */
- private final Collection<PlanNode> allNodes;
-
- /** The original program. */
- private final Plan originalProgram;
-
- /** Name of the job */
- private final String jobName;
-
- /**
- * Creates a new instance of this optimizer plan container. The plan is given and fully
- * described by the data sources, sinks and the collection of all nodes.
- *
- * @param sources The data sources.
- * @param sinks The data sinks.
- * @param allNodes A collection containing all nodes in the plan.
- * @param jobName The name of the program
- */
- public OptimizedPlan(Collection<SourcePlanNode> sources, Collection<SinkPlanNode> sinks,
- Collection<PlanNode> allNodes, String jobName, Plan programPlan)
- {
- this.dataSources = sources;
- this.dataSinks = sinks;
- this.allNodes = allNodes;
- this.jobName = jobName;
- this.originalProgram = programPlan;
- }
-
- /**
- * Gets the data sources from this OptimizedPlan.
- *
- * @return The data sources.
- */
- public Collection<SourcePlanNode> getDataSources() {
- return dataSources;
- }
-
- /**
- * Gets the data sinks from this OptimizedPlan.
- *
- * @return The data sinks.
- */
- public Collection<SinkPlanNode> getDataSinks() {
- return dataSinks;
- }
-
- /**
- * Gets all the nodes from this OptimizedPlan.
- *
- * @return All nodes.
- */
- public Collection<PlanNode> getAllNodes() {
- return allNodes;
- }
-
- /**
- * Returns the name of the program.
- *
- * @return The name of the program.
- */
- public String getJobName() {
- return this.jobName;
- }
-
- /**
- * Gets the original program plan from which this optimized plan was created.
- *
- * @return The original program plan.
- */
- public Plan getOriginalPactPlan() {
- return this.originalProgram;
- }
-
- // ------------------------------------------------------------------------
-
- /**
- * Applies the given visitor top down to all nodes, starting at the sinks.
- *
- * @param visitor
- * The visitor to apply to the nodes in this plan.
- * @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
- */
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- for (SinkPlanNode node : this.dataSinks) {
- node.accept(visitor);
- }
- }
-}
[52/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Rename
Pact* and Nephele* classes
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/PactCompiler.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/PactCompiler.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/PactCompiler.java
deleted file mode 100644
index d77fe1e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/PactCompiler.java
+++ /dev/null
@@ -1,1372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
-import org.apache.flink.optimizer.dag.GroupCombineNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
-import org.apache.flink.optimizer.dag.SortPartitionNode;
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Union;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.base.CrossOperatorBase;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.api.common.operators.base.FilterOperatorBase;
-import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.BulkIterationBase.PartialSolutionPlaceHolder;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.costs.DefaultCostEstimator;
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dag.BulkIterationNode;
-import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
-import org.apache.flink.optimizer.dag.CoGroupNode;
-import org.apache.flink.optimizer.dag.CollectorMapNode;
-import org.apache.flink.optimizer.dag.CrossNode;
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dag.FilterNode;
-import org.apache.flink.optimizer.dag.FlatMapNode;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.optimizer.dag.MapNode;
-import org.apache.flink.optimizer.dag.MapPartitionNode;
-import org.apache.flink.optimizer.dag.JoinNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.PactConnection;
-import org.apache.flink.optimizer.dag.PartitionNode;
-import org.apache.flink.optimizer.dag.ReduceNode;
-import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.optimizer.dag.SolutionSetNode;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.optimizer.dag.WorksetNode;
-import org.apache.flink.optimizer.deadlockdetect.DeadlockPreventer;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
-import org.apache.flink.optimizer.postpass.OptimizerPostPass;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.Visitor;
-
-/**
- * The optimizer that takes the user specified program plan and creates an optimized plan that contains
- * exact descriptions about how the physical execution will take place. It first translates the user
- * program into an internal optimizer representation and then chooses between different alternatives
- * for shipping strategies and local strategies.
- * <p>
- * The basic principle is taken from optimizer works in systems such as Volcano/Cascades and Selinger/System-R/DB2. The
- * optimizer walks from the sinks down, generating interesting properties, and ascends from the sources generating
- * alternative plans, pruning against the interesting properties.
- * <p>
- * The optimizer also assigns the memory to the individual tasks. This is currently done in a very simple fashion: All
- * sub-tasks that need memory (e.g. reduce or join) are given an equal share of memory.
- */
-public class PactCompiler {
-
- // ------------------------------------------------------------------------
- // Constants
- // ------------------------------------------------------------------------
-
- /**
- * Compiler hint key for the input channel's shipping strategy. This String is a key to the operator's stub
- * parameters. The corresponding value tells the compiler which shipping strategy to use for the input channel.
- * If the operator has two input channels, the shipping strategy is applied to both input channels.
- */
- public static final String HINT_SHIP_STRATEGY = "INPUT_SHIP_STRATEGY";
-
- /**
- * Compiler hint key for the <b>first</b> input channel's shipping strategy. This String is a key to
- * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
- * to use for the <b>first</b> input channel. Only applicable to operators with two inputs.
- */
- public static final String HINT_SHIP_STRATEGY_FIRST_INPUT = "INPUT_LEFT_SHIP_STRATEGY";
-
- /**
- * Compiler hint key for the <b>second</b> input channel's shipping strategy. This String is a key to
- * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
- * to use for the <b>second</b> input channel. Only applicable to operators with two inputs.
- */
- public static final String HINT_SHIP_STRATEGY_SECOND_INPUT = "INPUT_RIGHT_SHIP_STRATEGY";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a <b>Forward</b> strategy on the
- * input channel, i.e. no redistribution of any kind.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_FORWARD = "SHIP_FORWARD";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a random repartition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION= "SHIP_REPARTITION";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a hash-partition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION_HASH = "SHIP_REPARTITION_HASH";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a range-partition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION_RANGE = "SHIP_REPARTITION_RANGE";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a <b>broadcast</b> strategy on the
- * input channel.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_BROADCAST = "SHIP_BROADCAST";
-
- /**
- * Compiler hint key for the operator's local strategy. This String is a key to the operator's stub
- * parameters. The corresponding value tells the compiler which local strategy to use to process the
- * data inside one partition.
- * <p>
- * This hint is ignored by operators that do not have a local strategy (such as <i>Map</i>), or by operators that
- * have no choice in their local strategy (such as <i>Cross</i>).
- */
- public static final String HINT_LOCAL_STRATEGY = "LOCAL_STRATEGY";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
- * For example, a <i>Reduce</i> operator will sort the data to group it.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT = "LOCAL_STRATEGY_SORT";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
- * During sorting a combine method is repeatedly applied to reduce the data volume.
- * For example, a <i>Reduce</i> operator will sort the data to group it.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_COMBINING_SORT = "LOCAL_STRATEGY_COMBINING_SORT";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy on both
- * inputs with subsequent merging of inputs.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE = "LOCAL_STRATEGY_SORT_BOTH_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
- * The first input is sorted, the second input is assumed to be sorted. After sorting both inputs are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE = "LOCAL_STRATEGY_SORT_FIRST_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
- * The second input is sorted, the first input is assumed to be sorted. After sorting both inputs are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE = "LOCAL_STRATEGY_SORT_SECOND_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>merge based</b> local strategy.
- * Both inputs are assumed to be sorted and are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_MERGE = "LOCAL_STRATEGY_MERGE";
-
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
- * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
- * matching keys. The <b>first</b> input will be used to build the hash table, the second input will be
- * used to probe the table.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST = "LOCAL_STRATEGY_HASH_BUILD_FIRST";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
- * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
- * matching keys. The <b>second</b> input will be used to build the hash table, the first input will be
- * used to probe the table.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND = "LOCAL_STRATEGY_HASH_BUILD_SECOND";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
- * Hence, the data of the first input will be is streamed though, while the data of the second input is stored on
- * disk
- * and repeatedly read.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
- * Hence, the data of the second input will be is streamed though, while the data of the first input is stored on
- * disk
- * and repeatedly read.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
- * Further more, the first input, being the outer side, will be processed in blocks, and for each block, the second
- * input,
- * being the inner side, will read repeatedly from disk.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
- * Further more, the second input, being the outer side, will be processed in blocks, and for each block, the first
- * input,
- * being the inner side, will read repeatedly from disk.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND";
-
- /**
- * The log handle that is used by the compiler to log messages.
- */
- public static final Logger LOG = LoggerFactory.getLogger(PactCompiler.class);
-
- // ------------------------------------------------------------------------
- // Members
- // ------------------------------------------------------------------------
-
- /**
- * The statistics object used to obtain statistics, such as input sizes,
- * for the cost estimation process.
- */
- private final DataStatistics statistics;
-
- /**
- * The cost estimator used by the compiler.
- */
- private final CostEstimator costEstimator;
-
- /**
- * The default degree of parallelism for jobs compiled by this compiler.
- */
- private int defaultDegreeOfParallelism;
-
-
- // ------------------------------------------------------------------------
- // Constructor & Setup
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new optimizer instance. The optimizer has no access to statistics about the
- * inputs and can hence not determine any properties. It will perform all optimization with
- * unknown sizes and hence use only the heuristic cost functions, which result in the selection
- * of the most robust execution strategies.
- */
- public PactCompiler() {
- this(null, new DefaultCostEstimator());
- }
-
- /**
- * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
- * Given those statistics, the optimizer can make better choices for the execution strategies.
- *
- * @param stats
- * The statistics to be used to determine the input properties.
- */
- public PactCompiler(DataStatistics stats) {
- this(stats, new DefaultCostEstimator());
- }
-
- /**
- * Creates a new optimizer instance. The optimizer has no access to statistics about the
- * inputs and can hence not determine any properties. It will perform all optimization with
- * unknown sizes and hence use only the heuristic cost functions, which result in the selection
- * of the most robust execution strategies.
- *
- * The optimizer uses the given cost estimator to compute the costs of the individual operations.
- *
- * @param estimator The cost estimator to use to cost the individual operations.
- */
- public PactCompiler(CostEstimator estimator) {
- this(null, estimator);
- }
-
- /**
- * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
- * Given those statistics, the optimizer can make better choices for the execution strategies.
- *
- * The optimizer uses the given cost estimator to compute the costs of the individual operations.
- *
- * @param stats
- * The statistics to be used to determine the input properties.
- * @param estimator
- * The <tt>CostEstimator</tt> to use to cost the individual operations.
- */
- public PactCompiler(DataStatistics stats, CostEstimator estimator) {
- this.statistics = stats;
- this.costEstimator = estimator;
-
- // determine the default parallelism
- this.defaultDegreeOfParallelism = GlobalConfiguration.getInteger(
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE);
-
- if (defaultDegreeOfParallelism < 1) {
- LOG.warn("Config value " + defaultDegreeOfParallelism + " for option "
- + ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE + " is invalid. Ignoring and using a value of 1.");
- this.defaultDegreeOfParallelism = 1;
- }
- }
-
- // ------------------------------------------------------------------------
- // Getters / Setters
- // ------------------------------------------------------------------------
-
- public int getDefaultDegreeOfParallelism() {
- return defaultDegreeOfParallelism;
- }
-
- public void setDefaultDegreeOfParallelism(int defaultDegreeOfParallelism) {
- if (defaultDegreeOfParallelism > 0) {
- this.defaultDegreeOfParallelism = defaultDegreeOfParallelism;
- } else {
- throw new IllegalArgumentException("Default parallelism cannot be zero or negative.");
- }
- }
-
- // ------------------------------------------------------------------------
- // Compilation
- // ------------------------------------------------------------------------
-
- /**
- * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned
- * and all channels have a shipping strategy assigned.
- *
- * For more details on the optimization phase, see the comments for
- * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}.
- *
- * @param program The program to be translated.
- * @return The optimized plan.
- *
- * @throws CompilerException
- * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
- * situation during the compilation process.
- */
- public OptimizedPlan compile(Plan program) throws CompilerException {
- final OptimizerPostPass postPasser = getPostPassFromPlan(program);
- return compile(program, postPasser);
- }
-
- /**
- * Translates the given program to an OptimizedPlan. The optimized plan describes for each operator
- * which strategy to use (such as hash join versus sort-merge join), what data exchange method to use
- * (local pipe forward, shuffle, broadcast), what exchange mode to use (pipelined, batch),
- * where to cache intermediate results, etc,
- *
- * The optimization happens in multiple phases:
- * <ol>
- * <li>Create optimizer dag implementation of the program.
- *
- * <tt>OptimizerNode</tt> representations of the PACTs, assign parallelism and compute size estimates.</li>
- * <li>Compute interesting properties and auxiliary structures.</li>
- * <li>Enumerate plan alternatives. This cannot be done in the same step as the interesting property computation (as
- * opposed to the Database approaches), because we support plans that are not trees.</li>
- * </ol>
- *
- * @param program The program to be translated.
- * @param postPasser The function to be used for post passing the optimizer's plan and setting the
- * data type specific serialization routines.
- * @return The optimized plan.
- *
- * @throws CompilerException
- * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
- * situation during the compilation process.
- */
- private OptimizedPlan compile(Plan program, OptimizerPostPass postPasser) throws CompilerException {
- if (program == null || postPasser == null) {
- throw new NullPointerException();
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Beginning compilation of program '" + program.getJobName() + '\'');
- }
-
- final ExecutionMode defaultDataExchangeMode = program.getExecutionConfig().getExecutionMode();
-
- final int defaultParallelism = program.getDefaultParallelism() > 0 ?
- program.getDefaultParallelism() : this.defaultDegreeOfParallelism;
-
- // log the default settings
- LOG.debug("Using a default parallelism of {}", defaultParallelism);
- LOG.debug("Using default data exchange mode {}", defaultDataExchangeMode);
-
- // the first step in the compilation is to create the optimizer plan representation
- // this step does the following:
- // 1) It creates an optimizer plan node for each operator
- // 2) It connects them via channels
- // 3) It looks for hints about local strategies and channel types and
- // sets the types and strategies accordingly
- // 4) It makes estimates about the data volume of the data sources and
- // propagates those estimates through the plan
-
- GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode);
- program.accept(graphCreator);
-
- // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
- // each until we have only a single root node. This allows to transparently deal with the nodes with
- // multiple outputs
- OptimizerNode rootNode;
- if (graphCreator.sinks.size() == 1) {
- rootNode = graphCreator.sinks.get(0);
- } else if (graphCreator.sinks.size() > 1) {
- Iterator<DataSinkNode> iter = graphCreator.sinks.iterator();
- rootNode = iter.next();
-
- while (iter.hasNext()) {
- rootNode = new SinkJoiner(rootNode, iter.next());
- }
- } else {
- throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
- }
-
- // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
- // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
- rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
-
- // We are dealing with operator DAGs, rather than operator trees.
- // That requires us to deviate at some points from the classical DB optimizer algorithms.
- // This step build some auxiliary structures to help track branches and joins in the DAG
- BranchesVisitor branchingVisitor = new BranchesVisitor();
- rootNode.accept(branchingVisitor);
-
- // Propagate the interesting properties top-down through the graph
- InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
- rootNode.accept(propsVisitor);
-
- // perform a sanity check: the root may not have any unclosed branches
- if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
- throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
- "track the re-joining of branches correctly.");
- }
-
- // the final step is now to generate the actual plan alternatives
- List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);
-
- if (bestPlan.size() != 1) {
- throw new CompilerException("Error in compiler: more than one best plan was created!");
- }
-
- // check if the best plan's root is a data sink (single sink plan)
- // if so, directly take it. if it is a sink joiner node, get its contained sinks
- PlanNode bestPlanRoot = bestPlan.get(0);
- List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);
-
- if (bestPlanRoot instanceof SinkPlanNode) {
- bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
- } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {
- ((SinkJoinerPlanNode) bestPlanRoot).getDataSinks(bestPlanSinks);
- }
-
- DeadlockPreventer dp = new DeadlockPreventer();
- dp.resolveDeadlocks(bestPlanSinks);
-
- // finalize the plan
- OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program);
-
- plan.accept(new BinaryUnionReplacer());
-
- // post pass the plan. this is the phase where the serialization and comparator code is set
- postPasser.postPass(plan);
-
- return plan;
- }
-
- /**
- * This function performs only the first step to the compilation process - the creation of the optimizer
- * representation of the plan. No estimations or enumerations of alternatives are done here.
- *
- * @param program The plan to generate the optimizer representation for.
- * @return The optimizer representation of the plan, as a collection of all data sinks
- * from the plan can be traversed.
- */
- public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
- GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
- program.accept(graphCreator);
- return graphCreator.sinks;
- }
-
- // ------------------------------------------------------------------------
- // Visitors for Compilation Traversals
- // ------------------------------------------------------------------------
-
- /**
- * This utility class performs the translation from the user specified program to the optimizer plan.
- * It works as a visitor that walks the user's job in a depth-first fashion. During the descend, it creates
- * an optimizer node for each operator, respectively data source or -sink. During the ascend, it connects
- * the nodes to the full graph.
- * <p>
- * This translator relies on the <code>setInputs</code> method in the nodes. As that method implements the size
- * estimation and the awareness for optimizer hints, the sizes will be properly estimated and the translated plan
- * already respects all optimizer hints.
- */
- public static final class GraphCreatingVisitor implements Visitor<Operator<?>> {
-
- private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
- // corresponding optimizer nodes
-
- private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
-
- private final int defaultParallelism; // the default degree of parallelism
-
- private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
-
- private final ExecutionMode defaultDataExchangeMode;
-
- private final boolean forceDOP;
-
-
- public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
- this(null, false, defaultParallelism, defaultDataExchangeMode, null);
- }
-
- private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
- ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
- if (closure == null){
- con2node = new HashMap<Operator<?>, OptimizerNode>();
- } else {
- con2node = closure;
- }
-
- this.sinks = new ArrayList<DataSinkNode>(2);
- this.defaultParallelism = defaultParallelism;
- this.parent = parent;
- this.defaultDataExchangeMode = dataExchangeMode;
- this.forceDOP = forceDOP;
- }
-
- public List<DataSinkNode> getSinks() {
- return sinks;
- }
-
- @SuppressWarnings("deprecation")
- @Override
- public boolean preVisit(Operator<?> c) {
- // check if we have been here before
- if (this.con2node.containsKey(c)) {
- return false;
- }
-
- final OptimizerNode n;
-
- // create a node for the operator (or sink or source) if we have not been here before
- if (c instanceof GenericDataSinkBase) {
- DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
- this.sinks.add(dsn);
- n = dsn;
- }
- else if (c instanceof GenericDataSourceBase) {
- n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
- }
- else if (c instanceof MapOperatorBase) {
- n = new MapNode((MapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof MapPartitionOperatorBase) {
- n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
- n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FlatMapOperatorBase) {
- n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FilterOperatorBase) {
- n = new FilterNode((FilterOperatorBase<?, ?>) c);
- }
- else if (c instanceof ReduceOperatorBase) {
- n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
- }
- else if (c instanceof GroupReduceOperatorBase) {
- n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof GroupCombineOperatorBase) {
- n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof JoinOperatorBase) {
- n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CoGroupOperatorBase) {
- n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CrossOperatorBase) {
- n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof BulkIterationBase) {
- n = new BulkIterationNode((BulkIterationBase<?>) c);
- }
- else if (c instanceof DeltaIterationBase) {
- n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
- }
- else if (c instanceof Union){
- n = new BinaryUnionNode((Union<?>) c);
- }
- else if (c instanceof PartitionOperatorBase) {
- n = new PartitionNode((PartitionOperatorBase<?>) c);
- }
- else if (c instanceof SortPartitionOperatorBase) {
- n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
- }
- else if (c instanceof PartialSolutionPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
- final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
- final BulkIterationNode containingIterationNode =
- (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
- n = p;
- }
- else if (c instanceof WorksetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- WorksetNode p = new WorksetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
- n = p;
- }
- else if (c instanceof SolutionSetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
- n = p;
- }
- else {
- throw new IllegalArgumentException("Unknown operator type: " + c);
- }
-
- this.con2node.put(c, n);
-
- // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
- // key-less reducer (all-reduce)
- if (n.getDegreeOfParallelism() < 1) {
- // set the degree of parallelism
- int par = c.getDegreeOfParallelism();
- if (par > 0) {
- if (this.forceDOP && par != this.defaultParallelism) {
- par = this.defaultParallelism;
- LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
- "currently fixed to the parallelism of the surrounding operator (the iteration).");
- }
- } else {
- par = this.defaultParallelism;
- }
- n.setDegreeOfParallelism(par);
- }
-
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> c) {
-
- OptimizerNode n = this.con2node.get(c);
-
- // first connect to the predecessors
- n.setInput(this.con2node, this.defaultDataExchangeMode);
- n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
-
- // if the node represents a bulk iteration, we recursively translate the data flow now
- if (n instanceof BulkIterationNode) {
- final BulkIterationNode iterNode = (BulkIterationNode) n;
- final BulkIterationBase<?> iter = iterNode.getIterationContract();
-
- // pass a copy of the no iterative part into the iteration translation,
- // in case the iteration references its closure
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
- iterNode.getDegreeOfParallelism(), defaultDataExchangeMode, closure);
-
- BulkPartialSolutionNode partialSolution;
-
- iter.getNextPartialSolution().accept(recursiveCreator);
-
- partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
- OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
- if (partialSolution == null) {
- throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
- }
-
-
- OptimizerNode terminationCriterion = null;
-
- if (iter.getTerminationCriterion() != null) {
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
-
- // no intermediate node yet, traverse from the termination criterion to build the missing parts
- if (terminationCriterion == null) {
- iter.getTerminationCriterion().accept(recursiveCreator);
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
- }
- }
-
- iterNode.setPartialSolution(partialSolution);
- iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(identifier);
- }
- else if (n instanceof WorksetIterationNode) {
- final WorksetIterationNode iterNode = (WorksetIterationNode) n;
- final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
-
- // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
- // One check is for free during the translation, we do the other check here as a pre-condition
- {
- StepFunctionValidator wsf = new StepFunctionValidator();
- iter.getNextWorkset().accept(wsf);
- if (!wsf.foundWorkset) {
- throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
- "This is a prerequisite in delta iterations.");
- }
- }
-
- // calculate the closure of the anonymous function
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
- this, true, iterNode.getDegreeOfParallelism(), defaultDataExchangeMode, closure);
-
- // descend from the solution set delta. check that it depends on both the workset
- // and the solution set. If it does depend on both, this descend should create both nodes
- iter.getSolutionSetDelta().accept(recursiveCreator);
-
- final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
-
- if (worksetNode == null) {
- throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
- "This is a prerequisite in delta iterations.");
- }
-
- iter.getNextWorkset().accept(recursiveCreator);
-
- SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
-
- if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
- solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
- }
- else {
- for (PactConnection conn : solutionSetNode.getOutgoingConnections()) {
- OptimizerNode successor = conn.getTarget();
-
- if (successor.getClass() == JoinNode.class) {
- // find out which input to the match the solution set is
- JoinNode mn = (JoinNode) successor;
- if (mn.getFirstPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(0);
- } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else if (successor.getClass() == CoGroupNode.class) {
- CoGroupNode cg = (CoGroupNode) successor;
- if (cg.getFirstPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(0);
- } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else {
- throw new InvalidProgramException(
- "Error: The only operations allowed on the solution set are Join and CoGroup.");
- }
- }
- }
-
- final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
- final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
-
- // set the step function nodes to the iteration node
- iterNode.setPartialSolution(solutionSetNode, worksetNode);
- iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(pathIdentifier);
- }
- }
- }
-
- private static final class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
-
- private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
-
- private final int costWeight;
-
- private StaticDynamicPathIdentifier(int costWeight) {
- this.costWeight = costWeight;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return this.seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- visitable.identifyDynamicPath(this.costWeight);
-
- // check that there is no nested iteration on the dynamic path
- if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
- throw new CompilerException("Nested iterations are currently not supported.");
- }
- }
- }
-
- /**
- * Simple visitor that sets the minimal guaranteed memory per task based on the amount of available memory,
- * the number of memory consumers, and on the task's degree of parallelism.
- */
- public static final class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
-
- private final DataStatistics statistics;
-
- private int id = 1;
-
- public IdAndEstimatesVisitor(DataStatistics statistics) {
- this.statistics = statistics;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return visitable.getId() == -1;
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- // the node ids
- visitable.initId(this.id++);
-
- // connections need to figure out their maximum path depths
- for (PactConnection conn : visitable.getIncomingConnections()) {
- conn.initMaxDepth();
- }
- for (PactConnection conn : visitable.getBroadcastConnections()) {
- conn.initMaxDepth();
- }
-
- // the estimates
- visitable.computeOutputEstimates(this.statistics);
-
- // if required, recurse into the step function
- if (visitable instanceof IterationNode) {
- ((IterationNode) visitable).acceptForStepFunction(this);
- }
- }
- }
-
- /**
- * Visitor that computes the interesting properties for each node in the plan. On its recursive
- * depth-first descend, it propagates all interesting properties top-down.
- */
- public static final class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
-
- private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
-
- /**
- * Creates a new visitor that computes the interesting properties for all nodes in the plan.
- * It uses the given cost estimator used to compute the maximal costs for an interesting property.
- *
- * @param estimator
- * The cost estimator to estimate the maximal costs for interesting properties.
- */
- public InterestingPropertyVisitor(CostEstimator estimator) {
- this.estimator = estimator;
- }
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- // The interesting properties must be computed on the descend. In case a node has multiple outputs,
- // that computation must happen during the last descend.
-
- if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
- node.computeUnionOfInterestingPropertiesFromSuccessors();
- node.computeInterestingPropertiesForInputs(this.estimator);
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {}
- }
-
- /**
- * On its re-ascend (post visit) this visitor, computes auxiliary maps that are needed to support plans
- * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
- * output into more than one other node).
- */
- public static final class BranchesVisitor implements Visitor<OptimizerNode> {
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- return node.getOpenBranches() == null;
- }
-
- @Override
- public void postVisit(OptimizerNode node) {
- if (node instanceof IterationNode) {
- ((IterationNode) node).acceptForStepFunction(this);
- }
-
- node.computeUnclosedBranchStack();
- }
- }
-
- /**
- * Finalization of the plan:
- * - The graph of nodes is double-linked (links from child to parent are inserted)
- * - If unions join static and dynamic paths, the cache is marked as a memory consumer
- * - Relative memory fractions are assigned to all nodes.
- * - All nodes are collected into a set.
- */
- private static final class PlanFinalizer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
-
- private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
-
- private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
-
- private final Deque<IterationPlanNode> stackOfIterationNodes;
-
- private int memoryConsumerWeights; // a counter of all memory consumers
-
- /**
- * Creates a new plan finalizer.
- */
- private PlanFinalizer() {
- this.allNodes = new HashSet<PlanNode>();
- this.sources = new ArrayList<SourcePlanNode>();
- this.sinks = new ArrayList<SinkPlanNode>();
- this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
- }
-
- private OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
- this.memoryConsumerWeights = 0;
-
- // traverse the graph
- for (SinkPlanNode node : sinks) {
- node.accept(this);
- }
-
- // assign the memory to each node
- if (this.memoryConsumerWeights > 0) {
- for (PlanNode node : this.allNodes) {
- // assign memory to the driver strategy of the node
- final int consumerWeight = node.getMemoryConsumerWeight();
- if (consumerWeight > 0) {
- final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
- node.setRelativeMemoryPerSubtask(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
- node.getPactContract().getName() + ".");
- }
- }
-
- // assign memory to the local and global strategies of the channels
- for (Channel c : node.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- final double relativeMem = 1.0 / this.memoryConsumerWeights;
- c.setRelativeMemoryLocalStrategy(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
- "instance of " + c + ".");
- }
- }
- if (c.getTempMode() != TempMode.NONE) {
- final double relativeMem = 1.0/ this.memoryConsumerWeights;
- c.setRelativeTempMemory(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
- "table for " + c + ".");
- }
- }
- }
- }
- }
- return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
- }
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- // if we come here again, prevent a further descend
- if (!this.allNodes.add(visitable)) {
- return false;
- }
-
- if (visitable instanceof SinkPlanNode) {
- this.sinks.add((SinkPlanNode) visitable);
- }
- else if (visitable instanceof SourcePlanNode) {
- this.sources.add((SourcePlanNode) visitable);
- }
- else if (visitable instanceof BinaryUnionPlanNode) {
- BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- if (unionNode.unionsStaticAndDynamicPath()) {
- unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
- }
- }
- else if (visitable instanceof BulkPartialSolutionPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
- }
- else if (visitable instanceof WorksetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
- else if (visitable instanceof SolutionSetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
-
- // double-connect the connections. previously, only parents knew their children, because
- // one child candidate could have been referenced by multiple parents.
- for (Channel conn : visitable.getInputs()) {
- conn.setTarget(visitable);
- conn.getSource().addOutgoingChannel(conn);
- }
-
- for (Channel c : visitable.getBroadcastInputs()) {
- c.setTarget(visitable);
- c.getSource().addOutgoingChannel(c);
- }
-
- // count the memory consumption
- this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
- for (Channel c : visitable.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
- for (Channel c : visitable.getBroadcastInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
-
- // pass the visitor to the iteraton's step function
- if (visitable instanceof IterationPlanNode) {
- // push the iteration node onto the stack
- final IterationPlanNode iterNode = (IterationPlanNode) visitable;
- this.stackOfIterationNodes.addLast(iterNode);
-
- // recurse
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
-
- // pop the iteration node from the stack
- this.stackOfIterationNodes.removeLast();
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {}
- }
-
- /**
- * A visitor that traverses the graph and collects cascading binary unions into a single n-ary
- * union operator. The exception is, when on of the union inputs is materialized, such as in the
- * static-code-path-cache in iterations.
- */
- private static final class BinaryUnionReplacer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (this.seenBefore.add(visitable)) {
- if (visitable instanceof IterationPlanNode) {
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
- }
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
-
- if (visitable instanceof BinaryUnionPlanNode) {
-
- final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- final Channel in1 = unionNode.getInput1();
- final Channel in2 = unionNode.getInput2();
-
- if (!unionNode.unionsStaticAndDynamicPath()) {
-
- // both on static path, or both on dynamic path. we can collapse them
- NAryUnionPlanNode newUnionNode;
-
- List<Channel> inputs = new ArrayList<Channel>();
- collect(in1, inputs);
- collect(in2, inputs);
-
- newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
- unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
-
- newUnionNode.setDegreeOfParallelism(unionNode.getDegreeOfParallelism());
-
- for (Channel c : inputs) {
- c.setTarget(newUnionNode);
- }
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- else {
- // union between the static and the dynamic path. we need to handle this for now
- // through a special union operator
-
- // make sure that the first input is the cached (static) and the second input is the dynamic
- if (in1.isOnDynamicPath()) {
- BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
-
- in1.setTarget(newUnionNode);
- in2.setTarget(newUnionNode);
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- }
- }
- }
-
- private void collect(Channel in, List<Channel> inputs) {
- if (in.getSource() instanceof NAryUnionPlanNode) {
- // sanity check
- if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
- throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
- }
- if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
- throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
- }
-
- inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
- } else {
- // is not a collapsed union node, so we take the channel directly
- inputs.add(in);
- }
- }
- }
-
- private static final class StepFunctionValidator implements Visitor<Operator<?>> {
-
- private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
-
- private boolean foundWorkset;
-
- @Override
- public boolean preVisit(Operator<?> visitable) {
- if (visitable instanceof WorksetPlaceHolder) {
- foundWorkset = true;
- }
-
- return (!foundWorkset) && seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(Operator<?> visitable) {}
- }
-
- // ------------------------------------------------------------------------
- // Miscellaneous
- // ------------------------------------------------------------------------
-
- private OptimizerPostPass getPostPassFromPlan(Plan program) {
- final String className = program.getPostPassClassName();
- if (className == null) {
- throw new CompilerException("Optimizer Post Pass class description is null");
- }
- try {
- Class<? extends OptimizerPostPass> clazz = Class.forName(className).asSubclass(OptimizerPostPass.class);
- try {
- return InstantiationUtil.instantiate(clazz, OptimizerPostPass.class);
- } catch (RuntimeException rtex) {
- // unwrap the source exception
- if (rtex.getCause() != null) {
- throw new CompilerException("Cannot instantiate optimizer post pass: " + rtex.getMessage(), rtex.getCause());
- } else {
- throw rtex;
- }
- }
- } catch (ClassNotFoundException cnfex) {
- throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex);
- } catch (ClassCastException ccex) {
- throw new CompilerException("Class '" + className + "' is not an optimizer post passer.", ccex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
index f8404c4..d199ae7 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/AbstractPartialSolutionNode.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.dag;
import java.util.Collections;
@@ -62,8 +61,8 @@ public abstract class AbstractPartialSolutionNode extends OptimizerNode {
}
@Override
- public List<PactConnection> getIncomingConnections() {
- return Collections.<PactConnection>emptyList();
+ public List<DagConnection> getIncomingConnections() {
+ return Collections.emptyList();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
index 8aed10c..068799e 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BinaryUnionNode.java
@@ -107,7 +107,7 @@ public class BinaryUnionNode extends TwoInputNode {
final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
- List<PactConnection> broadcastConnections = getBroadcastConnections();
+ List<DagConnection> broadcastConnections = getBroadcastConnections();
if (broadcastConnections != null && broadcastConnections.size() > 0) {
throw new CompilerException("Found BroadcastVariables on a Union operation");
}
@@ -122,9 +122,9 @@ public class BinaryUnionNode extends TwoInputNode {
final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
- final int dop = getDegreeOfParallelism();
- final int inDop1 = getFirstPredecessorNode().getDegreeOfParallelism();
- final int inDop2 = getSecondPredecessorNode().getDegreeOfParallelism();
+ final int dop = getParallelism();
+ final int inDop1 = getFirstPredecessorNode().getParallelism();
+ final int inDop2 = getSecondPredecessorNode().getParallelism();
final boolean dopChange1 = dop != inDop1;
final boolean dopChange2 = dop != inDop2;
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
index 8112748..c55d17a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
@@ -31,7 +31,7 @@ import org.apache.flink.api.common.operators.base.BulkIterationBase;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.Optimizer.InterestingPropertyVisitor;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dag.WorksetIterationNode.SingleRootJoiner;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
@@ -62,9 +62,9 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
private OptimizerNode nextPartialSolution;
- private PactConnection rootConnection; // connection out of the next partial solution
+ private DagConnection rootConnection; // connection out of the next partial solution
- private PactConnection terminationCriterionRootConnection; // connection out of the term. criterion
+ private DagConnection terminationCriterionRootConnection; // connection out of the term. criterion
private OptimizerNode singleRoot;
@@ -93,7 +93,7 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
// --------------------------------------------------------------------------------------------
public BulkIterationBase<?> getIterationContract() {
- return (BulkIterationBase<?>) getPactContract();
+ return (BulkIterationBase<?>) getOperator();
}
/**
@@ -133,14 +133,14 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
// check if the root of the step function has the same DOP as the iteration
// or if the step function has any operator at all
- if (nextPartialSolution.getDegreeOfParallelism() != getDegreeOfParallelism() ||
+ if (nextPartialSolution.getParallelism() != getParallelism() ||
nextPartialSolution == partialSolution || nextPartialSolution instanceof BinaryUnionNode)
{
// add a no-op to the root to express the re-partitioning
NoOpNode noop = new NoOpNode();
- noop.setDegreeOfParallelism(getDegreeOfParallelism());
+ noop.setDegreeOfParallelism(getParallelism());
- PactConnection noOpConn = new PactConnection(nextPartialSolution, noop, ExecutionMode.PIPELINED);
+ DagConnection noOpConn = new DagConnection(nextPartialSolution, noop, ExecutionMode.PIPELINED);
noop.setIncomingConnection(noOpConn);
nextPartialSolution.addOutgoingConnection(noOpConn);
@@ -152,13 +152,13 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
if (terminationCriterion == null) {
this.singleRoot = nextPartialSolution;
- this.rootConnection = new PactConnection(nextPartialSolution, ExecutionMode.PIPELINED);
+ this.rootConnection = new DagConnection(nextPartialSolution, ExecutionMode.PIPELINED);
}
else {
// we have a termination criterion
SingleRootJoiner singleRootJoiner = new SingleRootJoiner();
- this.rootConnection = new PactConnection(nextPartialSolution, singleRootJoiner, ExecutionMode.PIPELINED);
- this.terminationCriterionRootConnection = new PactConnection(terminationCriterion, singleRootJoiner,
+ this.rootConnection = new DagConnection(nextPartialSolution, singleRootJoiner, ExecutionMode.PIPELINED);
+ this.terminationCriterionRootConnection = new DagConnection(terminationCriterion, singleRootJoiner,
ExecutionMode.PIPELINED);
singleRootJoiner.setInputs(this.rootConnection, this.terminationCriterionRootConnection);
@@ -323,7 +323,7 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
locPropsReq.parameterizeChannel(toNoOp);
UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
- rebuildPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
+ rebuildPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
@@ -352,7 +352,7 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
// 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
if (terminationCriterion == null) {
for (PlanNode candidate : candidates) {
- BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
+ BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate);
GlobalProperties gProps = candidate.getGlobalProperties().clone();
LocalProperties lProps = candidate.getLocalProperties().clone();
node.initProperties(gProps, lProps);
@@ -367,7 +367,7 @@ public class BulkIterationNode extends SingleInputNode implements IterationNode
for (PlanNode candidate : candidates) {
for (PlanNode terminationCandidate : terminationCriterionCandidates) {
if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
- BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
+ BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate);
GlobalProperties gProps = candidate.getGlobalProperties().clone();
LocalProperties lProps = candidate.getLocalProperties().clone();
node.initProperties(gProps, lProps);
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
index a6e03ff..25a7eef 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkPartialSolutionNode.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.dag;
import java.util.Collections;
@@ -49,7 +48,8 @@ public class BulkPartialSolutionNode extends AbstractPartialSolutionNode {
if (this.cachedPlans != null) {
throw new IllegalStateException();
} else {
- this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this, "PartialSolution ("+this.getPactContract().getName()+")", gProps, lProps, initialInput));
+ this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
+ "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
}
}
@@ -73,13 +73,14 @@ public class BulkPartialSolutionNode extends AbstractPartialSolutionNode {
// --------------------------------------------------------------------------------------------
/**
- * Gets the contract object for this data source node.
+ * Gets the operator (here the {@link PartialSolutionPlaceHolder}) that is represented by this
+ * optimizer node.
*
- * @return The contract.
+ * @return The operator represented by this optimizer node.
*/
@Override
- public PartialSolutionPlaceHolder<?> getPactContract() {
- return (PartialSolutionPlaceHolder<?>) super.getPactContract();
+ public PartialSolutionPlaceHolder<?> getOperator() {
+ return (PartialSolutionPlaceHolder<?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
index 7c0cc9a..92076c3 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CoGroupNode.java
@@ -50,8 +50,8 @@ public class CoGroupNode extends TwoInputNode {
* @return The CoGroup operator.
*/
@Override
- public CoGroupOperatorBase<?, ?, ?, ?> getPactContract() {
- return (CoGroupOperatorBase<?, ?, ?, ?>) super.getPactContract();
+ public CoGroupOperatorBase<?, ?, ?, ?> getOperator() {
+ return (CoGroupOperatorBase<?, ?, ?, ?>) super.getOperator();
}
@Override
@@ -85,7 +85,7 @@ public class CoGroupNode extends TwoInputNode {
Ordering groupOrder1 = null;
Ordering groupOrder2 = null;
- CoGroupOperatorBase<?, ?, ?, ?> cgc = getPactContract();
+ CoGroupOperatorBase<?, ?, ?, ?> cgc = getOperator();
groupOrder1 = cgc.getGroupOrderForInputOne();
groupOrder2 = cgc.getGroupOrderForInputTwo();
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
index afeed1d..8de67e8 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/CrossNode.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.operators.base.CrossOperatorBase;
import org.apache.flink.api.common.operators.base.CrossOperatorBase.CrossHint;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.PactCompiler;
+import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.operators.CrossBlockOuterFirstDescriptor;
import org.apache.flink.optimizer.operators.CrossBlockOuterSecondDescriptor;
import org.apache.flink.optimizer.operators.CrossStreamOuterFirstDescriptor;
@@ -50,7 +50,7 @@ public class CrossNode extends TwoInputNode {
super(operation);
Configuration conf = operation.getParameters();
- String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
+ String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
CrossHint hint = operation.getCrossHint();
@@ -60,13 +60,13 @@ public class CrossNode extends TwoInputNode {
final boolean allowBCsecond = hint != CrossHint.FIRST_IS_SMALL;
final OperatorDescriptorDual fixedDriverStrat;
- if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
+ if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
fixedDriverStrat = new CrossBlockOuterFirstDescriptor(allowBCfirst, allowBCsecond);
- } else if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND.equals(localStrategy)) {
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND.equals(localStrategy)) {
fixedDriverStrat = new CrossBlockOuterSecondDescriptor(allowBCfirst, allowBCsecond);
- } else if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST.equals(localStrategy)) {
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST.equals(localStrategy)) {
fixedDriverStrat = new CrossStreamOuterFirstDescriptor(allowBCfirst, allowBCsecond);
- } else if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND.equals(localStrategy)) {
+ } else if (Optimizer.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND.equals(localStrategy)) {
fixedDriverStrat = new CrossStreamOuterSecondDescriptor(allowBCfirst, allowBCsecond);
} else {
throw new CompilerException("Invalid local strategy hint for cross contract: " + localStrategy);
@@ -99,8 +99,8 @@ public class CrossNode extends TwoInputNode {
// ------------------------------------------------------------------------
@Override
- public CrossOperatorBase<?, ?, ?, ?> getPactContract() {
- return (CrossOperatorBase<?, ?, ?, ?>) super.getPactContract();
+ public CrossOperatorBase<?, ?, ?, ?> getOperator() {
+ return (CrossOperatorBase<?, ?, ?, ?>) super.getOperator();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
new file mode 100644
index 0000000..360f579
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.plandump.DumpableConnection;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+/**
+ * A connection between to operators. Represents an intermediate result
+ * and a data exchange between the two operators.
+ *
+ * The data exchange has a mode in which it performs (batch / pipelined).
+ *
+ * The data exchange strategy may be set on this connection, in which case
+ * it is fixed and will not be determined during candidate plan enumeration.
+ *
+ * During the enumeration of interesting properties, this connection also holds
+ * all interesting properties generated by the successor operator.
+ */
+public class DagConnection implements EstimateProvider, DumpableConnection<OptimizerNode> {
+
+ private final OptimizerNode source; // The source node of the connection
+
+ private final OptimizerNode target; // The target node of the connection.
+
+ private final ExecutionMode dataExchangeMode; // defines whether to use batch or pipelined data exchange
+
+ private InterestingProperties interestingProps; // local properties that succeeding nodes are interested in
+
+ private ShipStrategyType shipStrategy; // The data shipping strategy, if predefined.
+
+ private TempMode materializationMode = TempMode.NONE; // the materialization mode
+
+ private int maxDepth = -1;
+
+ private boolean breakPipeline; // whether this connection should break the pipeline due to potential deadlocks
+
+ /**
+ * Creates a new Connection between two nodes. The shipping strategy is by default <tt>NONE</tt>.
+ * The temp mode is by default <tt>NONE</tt>.
+ *
+ * @param source
+ * The source node.
+ * @param target
+ * The target node.
+ */
+ public DagConnection(OptimizerNode source, OptimizerNode target, ExecutionMode exchangeMode) {
+ this(source, target, null, exchangeMode);
+ }
+
+ /**
+ * Creates a new Connection between two nodes.
+ *
+ * @param source
+ * The source node.
+ * @param target
+ * The target node.
+ * @param shipStrategy
+ * The shipping strategy.
+ * @param exchangeMode
+ * The data exchange mode (pipelined / batch / batch only for shuffles / ... )
+ */
+ public DagConnection(OptimizerNode source, OptimizerNode target,
+ ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
+ {
+ if (source == null || target == null) {
+ throw new NullPointerException("Source and target must not be null.");
+ }
+ this.source = source;
+ this.target = target;
+ this.shipStrategy = shipStrategy;
+ this.dataExchangeMode = exchangeMode;
+ }
+
+ /**
+ * Constructor to create a result from an operator that is not
+ * consumed by another operator.
+ *
+ * @param source
+ * The source node.
+ */
+ public DagConnection(OptimizerNode source, ExecutionMode exchangeMode) {
+ if (source == null) {
+ throw new NullPointerException("Source and target must not be null.");
+ }
+ this.source = source;
+ this.target = null;
+ this.shipStrategy = ShipStrategyType.NONE;
+ this.dataExchangeMode = exchangeMode;
+ }
+
+ /**
+ * Gets the source of the connection.
+ *
+ * @return The source Node.
+ */
+ public OptimizerNode getSource() {
+ return this.source;
+ }
+
+ /**
+ * Gets the target of the connection.
+ *
+ * @return The target node.
+ */
+ public OptimizerNode getTarget() {
+ return this.target;
+ }
+
+ /**
+ * Gets the shipping strategy for this connection.
+ *
+ * @return The connection's shipping strategy.
+ */
+ public ShipStrategyType getShipStrategy() {
+ return this.shipStrategy;
+ }
+
+ /**
+ * Sets the shipping strategy for this connection.
+ *
+ * @param strategy
+ * The shipping strategy to be applied to this connection.
+ */
+ public void setShipStrategy(ShipStrategyType strategy) {
+ this.shipStrategy = strategy;
+ }
+
+ /**
+ * Gets the data exchange mode to use for this connection.
+ *
+ * @return The data exchange mode to use for this connection.
+ */
+ public ExecutionMode getDataExchangeMode() {
+ if (dataExchangeMode == null) {
+ throw new IllegalStateException("This connection does not have the data exchange mode set");
+ }
+ return dataExchangeMode;
+ }
+
+ /**
+ * Marks that this connection should do a decoupled data exchange (such as batched)
+ * rather then pipeline data. Connections are marked as pipeline breakers to avoid
+ * deadlock situations.
+ */
+ public void markBreaksPipeline() {
+ this.breakPipeline = true;
+ }
+
+ /**
+ * Checks whether this connection is marked to break the pipeline.
+ *
+ * @return True, if this connection is marked to break the pipeline, false otherwise.
+ */
+ public boolean isBreakingPipeline() {
+ return this.breakPipeline;
+ }
+
+ /**
+ * Gets the interesting properties object for this pact connection.
+ * If the interesting properties for this connections have not yet been set,
+ * this method returns null.
+ *
+ * @return The collection of all interesting properties, or null, if they have not yet been set.
+ */
+ public InterestingProperties getInterestingProperties() {
+ return this.interestingProps;
+ }
+
+ /**
+ * Sets the interesting properties for this pact connection.
+ *
+ * @param props The interesting properties.
+ */
+ public void setInterestingProperties(InterestingProperties props) {
+ if (this.interestingProps == null) {
+ this.interestingProps = props;
+ } else {
+ throw new IllegalStateException("Interesting Properties have already been set.");
+ }
+ }
+
+ public void clearInterestingProperties() {
+ this.interestingProps = null;
+ }
+
+ public void initMaxDepth() {
+
+ if (this.maxDepth == -1) {
+ this.maxDepth = this.source.getMaxDepth() + 1;
+ } else {
+ throw new IllegalStateException("Maximum path depth has already been initialized.");
+ }
+ }
+
+ public int getMaxDepth() {
+ if (this.maxDepth != -1) {
+ return this.maxDepth;
+ } else {
+ throw new IllegalStateException("Maximum path depth has not been initialized.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Estimates
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public long getEstimatedOutputSize() {
+ return this.source.getEstimatedOutputSize();
+ }
+
+ @Override
+ public long getEstimatedNumRecords() {
+ return this.source.getEstimatedNumRecords();
+ }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() {
+ return this.source.getEstimatedAvgWidthPerOutputRecord();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ public TempMode getMaterializationMode() {
+ return this.materializationMode;
+ }
+
+ public void setMaterializationMode(TempMode materializationMode) {
+ this.materializationMode = materializationMode;
+ }
+
+ public boolean isOnDynamicPath() {
+ return this.source.isOnDynamicPath();
+ }
+
+ public int getCostWeight() {
+ return this.source.getCostWeight();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder(50);
+ buf.append("Connection: ");
+
+ if (this.source == null) {
+ buf.append("null");
+ } else {
+ buf.append(this.source.getOperator().getName());
+ buf.append('(').append(this.source.getName()).append(')');
+ }
+
+ buf.append(" -> ");
+
+ if (this.shipStrategy != null) {
+ buf.append('[');
+ buf.append(this.shipStrategy.name());
+ buf.append(']').append(' ');
+ }
+
+ if (this.target == null) {
+ buf.append("null");
+ } else {
+ buf.append(this.target.getOperator().getName());
+ buf.append('(').append(this.target.getName()).append(')');
+ }
+
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a9150b30/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
index 9e4f457..dbe04f4 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DataSinkNode.java
@@ -46,7 +46,7 @@ import org.apache.flink.util.Visitor;
*/
public class DataSinkNode extends OptimizerNode {
- protected PactConnection input; // The input edge
+ protected DagConnection input; // The input edge
/**
* Creates a new DataSinkNode for the given sink operator.
@@ -64,7 +64,7 @@ public class DataSinkNode extends OptimizerNode {
*
* @return The input connection.
*/
- public PactConnection getInputConnection() {
+ public DagConnection getInputConnection() {
return this.input;
}
@@ -87,8 +87,8 @@ public class DataSinkNode extends OptimizerNode {
* @return The node's underlying operator.
*/
@Override
- public GenericDataSinkBase<?> getPactContract() {
- return (GenericDataSinkBase<?>) super.getPactContract();
+ public GenericDataSinkBase<?> getOperator() {
+ return (GenericDataSinkBase<?>) super.getOperator();
}
@Override
@@ -97,7 +97,7 @@ public class DataSinkNode extends OptimizerNode {
}
@Override
- public List<PactConnection> getIncomingConnections() {
+ public List<DagConnection> getIncomingConnections() {
return Collections.singletonList(this.input);
}
@@ -107,19 +107,19 @@ public class DataSinkNode extends OptimizerNode {
* @return An empty list.
*/
@Override
- public List<PactConnection> getOutgoingConnections() {
+ public List<DagConnection> getOutgoingConnections() {
return Collections.emptyList();
}
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
- Operator<?> children = getPactContract().getInput();
+ Operator<?> children = getOperator().getInput();
final OptimizerNode pred;
- final PactConnection conn;
+ final DagConnection conn;
pred = contractToNode.get(children);
- conn = new PactConnection(pred, this, defaultExchangeMode);
+ conn = new DagConnection(pred, this, defaultExchangeMode);
// create the connection and add it
this.input = conn;
@@ -141,8 +141,8 @@ public class DataSinkNode extends OptimizerNode {
final InterestingProperties iProps = new InterestingProperties();
{
- final Ordering partitioning = getPactContract().getPartitionOrdering();
- final DataDistribution dataDist = getPactContract().getDataDistribution();
+ final Ordering partitioning = getOperator().getPartitionOrdering();
+ final DataDistribution dataDist = getOperator().getDataDistribution();
final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
if (partitioning != null) {
if(dataDist != null) {
@@ -156,7 +156,7 @@ public class DataSinkNode extends OptimizerNode {
}
{
- final Ordering localOrder = getPactContract().getLocalOrder();
+ final Ordering localOrder = getOperator().getLocalOrder();
final RequestedLocalProperties orderProps = new RequestedLocalProperties();
if (localOrder != null) {
orderProps.setOrdering(localOrder);
@@ -184,7 +184,7 @@ public class DataSinkNode extends OptimizerNode {
}
@Override
- protected List<UnclosedBranchDescriptor> getBranchesForParent(PactConnection parent) {
+ protected List<UnclosedBranchDescriptor> getBranchesForParent(DagConnection parent) {
// return our own stack of open branches, because nothing is added
return this.openBranches;
}
@@ -204,8 +204,8 @@ public class DataSinkNode extends OptimizerNode {
List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
List<PlanNode> outputPlans = new ArrayList<PlanNode>();
- final int dop = getDegreeOfParallelism();
- final int inDop = getPredecessorNode().getDegreeOfParallelism();
+ final int dop = getParallelism();
+ final int inDop = getPredecessorNode().getParallelism();
final ExecutionMode executionMode = this.input.getDataExchangeMode();
final boolean dopChange = dop != inDop;
@@ -224,7 +224,7 @@ public class DataSinkNode extends OptimizerNode {
// no need to check whether the created properties meet what we need in case
// of ordering or global ordering, because the only interesting properties we have
// are what we require
- outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getPactContract().getName()+")" ,c));
+ outputPlans.add(new SinkPlanNode(this, "DataSink ("+this.getOperator().getName()+")" ,c));
}
}
}
[45/53] [abbrv] flink git commit: [optimizer] Moved optimizer graph
traversals to dedicated classes to simplify Optimizer class.
Posted by se...@apache.org.
[optimizer] Moved optimizer graph traversals to dedicated classes to simplify Optimizer class.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/5757850c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/5757850c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/5757850c
Branch: refs/heads/master
Commit: 5757850ce1df5684e4c391d77ce033fd54afd0af
Parents: c2db181
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 16:43:28 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
.../org/apache/flink/optimizer/Optimizer.java | 839 +------------------
.../flink/optimizer/dag/BulkIterationNode.java | 2 +-
.../flink/optimizer/dag/DagConnection.java | 2 +-
.../optimizer/dag/WorksetIterationNode.java | 2 +-
.../traversals/BinaryUnionReplacer.java | 125 +++
.../optimizer/traversals/BranchesVisitor.java | 46 +
.../traversals/GraphCreatingVisitor.java | 392 +++++++++
.../traversals/IdAndEstimatesVisitor.java | 68 ++
.../traversals/InterestingPropertyVisitor.java | 60 ++
.../optimizer/traversals/PlanFinalizer.java | 229 +++++
.../traversals/StaticDynamicPathIdentifier.java | 58 ++
.../traversals/StepFunctionValidator.java | 53 ++
.../optimizer/traversals/package-info.java | 27 +
.../dataexchange/PipelineBreakingTest.java | 12 +-
14 files changed, 1087 insertions(+), 828 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
index e76501c..2101428 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
@@ -18,94 +18,35 @@
package org.apache.flink.optimizer;
-import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
-import org.apache.flink.optimizer.dag.GroupCombineNode;
+import org.apache.flink.optimizer.traversals.BinaryUnionReplacer;
+import org.apache.flink.optimizer.traversals.BranchesVisitor;
+import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
+import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.traversals.PlanFinalizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
-import org.apache.flink.optimizer.dag.SortPartitionNode;
-import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.operators.GenericDataSinkBase;
-import org.apache.flink.api.common.operators.GenericDataSourceBase;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Union;
-import org.apache.flink.api.common.operators.base.BulkIterationBase;
-import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.base.CrossOperatorBase;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase;
-import org.apache.flink.api.common.operators.base.FilterOperatorBase;
-import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.BulkIterationBase.PartialSolutionPlaceHolder;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
-import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.costs.DefaultCostEstimator;
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dag.BulkIterationNode;
-import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
-import org.apache.flink.optimizer.dag.CoGroupNode;
-import org.apache.flink.optimizer.dag.CollectorMapNode;
-import org.apache.flink.optimizer.dag.CrossNode;
import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dag.FilterNode;
-import org.apache.flink.optimizer.dag.FlatMapNode;
-import org.apache.flink.optimizer.dag.GroupReduceNode;
-import org.apache.flink.optimizer.dag.IterationNode;
-import org.apache.flink.optimizer.dag.MapNode;
-import org.apache.flink.optimizer.dag.MapPartitionNode;
-import org.apache.flink.optimizer.dag.JoinNode;
import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.DagConnection;
-import org.apache.flink.optimizer.dag.PartitionNode;
-import org.apache.flink.optimizer.dag.ReduceNode;
import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.optimizer.dag.SolutionSetNode;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.optimizer.dag.WorksetNode;
import org.apache.flink.optimizer.deadlockdetect.DeadlockPreventer;
-import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.IterationPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.PlanNode;
import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.plan.WorksetPlanNode;
import org.apache.flink.optimizer.postpass.OptimizerPostPass;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.Visitor;
/**
* The optimizer that takes the user specified program plan and creates an optimized plan that contains
@@ -516,16 +457,18 @@ public class Optimizer {
// each until we have only a single root node. This allows to transparently deal with the nodes with
// multiple outputs
OptimizerNode rootNode;
- if (graphCreator.sinks.size() == 1) {
- rootNode = graphCreator.sinks.get(0);
- } else if (graphCreator.sinks.size() > 1) {
- Iterator<DataSinkNode> iter = graphCreator.sinks.iterator();
+ if (graphCreator.getSinks().size() == 1) {
+ rootNode = graphCreator.getSinks().get(0);
+ }
+ else if (graphCreator.getSinks().size() > 1) {
+ Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator();
rootNode = iter.next();
while (iter.hasNext()) {
rootNode = new SinkJoiner(rootNode, iter.next());
}
- } else {
+ }
+ else {
throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
}
@@ -592,755 +535,9 @@ public class Optimizer {
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
program.accept(graphCreator);
- return graphCreator.sinks;
+ return graphCreator.getSinks();
}
-
- // ------------------------------------------------------------------------
- // Visitors for Compilation Traversals
- // ------------------------------------------------------------------------
-
- /**
- * This utility class performs the translation from the user specified program to the optimizer plan.
- * It works as a visitor that walks the user's job in a depth-first fashion. During the descend, it creates
- * an optimizer node for each operator, respectively data source or -sink. During the ascend, it connects
- * the nodes to the full graph.
- * <p>
- * This translator relies on the <code>setInputs</code> method in the nodes. As that method implements the size
- * estimation and the awareness for optimizer hints, the sizes will be properly estimated and the translated plan
- * already respects all optimizer hints.
- */
- public static final class GraphCreatingVisitor implements Visitor<Operator<?>> {
-
- private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
- // corresponding optimizer nodes
-
- private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
-
- private final int defaultParallelism; // the default degree of parallelism
-
- private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
-
- private final ExecutionMode defaultDataExchangeMode;
-
- private final boolean forceDOP;
-
-
- public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
- this(null, false, defaultParallelism, defaultDataExchangeMode, null);
- }
-
- private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
- ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
- if (closure == null){
- con2node = new HashMap<Operator<?>, OptimizerNode>();
- } else {
- con2node = closure;
- }
-
- this.sinks = new ArrayList<DataSinkNode>(2);
- this.defaultParallelism = defaultParallelism;
- this.parent = parent;
- this.defaultDataExchangeMode = dataExchangeMode;
- this.forceDOP = forceDOP;
- }
-
- public List<DataSinkNode> getSinks() {
- return sinks;
- }
- @SuppressWarnings("deprecation")
- @Override
- public boolean preVisit(Operator<?> c) {
- // check if we have been here before
- if (this.con2node.containsKey(c)) {
- return false;
- }
-
- final OptimizerNode n;
-
- // create a node for the operator (or sink or source) if we have not been here before
- if (c instanceof GenericDataSinkBase) {
- DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
- this.sinks.add(dsn);
- n = dsn;
- }
- else if (c instanceof GenericDataSourceBase) {
- n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
- }
- else if (c instanceof MapOperatorBase) {
- n = new MapNode((MapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof MapPartitionOperatorBase) {
- n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
- n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FlatMapOperatorBase) {
- n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof FilterOperatorBase) {
- n = new FilterNode((FilterOperatorBase<?, ?>) c);
- }
- else if (c instanceof ReduceOperatorBase) {
- n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
- }
- else if (c instanceof GroupReduceOperatorBase) {
- n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof GroupCombineOperatorBase) {
- n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
- }
- else if (c instanceof JoinOperatorBase) {
- n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CoGroupOperatorBase) {
- n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof CrossOperatorBase) {
- n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
- }
- else if (c instanceof BulkIterationBase) {
- n = new BulkIterationNode((BulkIterationBase<?>) c);
- }
- else if (c instanceof DeltaIterationBase) {
- n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
- }
- else if (c instanceof Union){
- n = new BinaryUnionNode((Union<?>) c);
- }
- else if (c instanceof PartitionOperatorBase) {
- n = new PartitionNode((PartitionOperatorBase<?>) c);
- }
- else if (c instanceof SortPartitionOperatorBase) {
- n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
- }
- else if (c instanceof PartialSolutionPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
- final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
- final BulkIterationNode containingIterationNode =
- (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else if (c instanceof WorksetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- WorksetNode p = new WorksetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else if (c instanceof SolutionSetPlaceHolder) {
- if (this.parent == null) {
- throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
- }
-
- final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
- final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
- final WorksetIterationNode containingIterationNode =
- (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
-
- // catch this for the recursive translation of step functions
- SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
- p.setDegreeOfParallelism(containingIterationNode.getParallelism());
- n = p;
- }
- else {
- throw new IllegalArgumentException("Unknown operator type: " + c);
- }
-
- this.con2node.put(c, n);
-
- // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
- // key-less reducer (all-reduce)
- if (n.getParallelism() < 1) {
- // set the degree of parallelism
- int par = c.getDegreeOfParallelism();
- if (par > 0) {
- if (this.forceDOP && par != this.defaultParallelism) {
- par = this.defaultParallelism;
- LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
- "currently fixed to the parallelism of the surrounding operator (the iteration).");
- }
- } else {
- par = this.defaultParallelism;
- }
- n.setDegreeOfParallelism(par);
- }
-
- return true;
- }
-
- @Override
- public void postVisit(Operator<?> c) {
-
- OptimizerNode n = this.con2node.get(c);
-
- // first connect to the predecessors
- n.setInput(this.con2node, this.defaultDataExchangeMode);
- n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
-
- // if the node represents a bulk iteration, we recursively translate the data flow now
- if (n instanceof BulkIterationNode) {
- final BulkIterationNode iterNode = (BulkIterationNode) n;
- final BulkIterationBase<?> iter = iterNode.getIterationContract();
-
- // pass a copy of the no iterative part into the iteration translation,
- // in case the iteration references its closure
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
- iterNode.getParallelism(), defaultDataExchangeMode, closure);
-
- BulkPartialSolutionNode partialSolution;
-
- iter.getNextPartialSolution().accept(recursiveCreator);
-
- partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
- OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
- if (partialSolution == null) {
- throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
- }
-
-
- OptimizerNode terminationCriterion = null;
-
- if (iter.getTerminationCriterion() != null) {
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
-
- // no intermediate node yet, traverse from the termination criterion to build the missing parts
- if (terminationCriterion == null) {
- iter.getTerminationCriterion().accept(recursiveCreator);
- terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
- }
- }
-
- iterNode.setPartialSolution(partialSolution);
- iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(identifier);
- }
- else if (n instanceof WorksetIterationNode) {
- final WorksetIterationNode iterNode = (WorksetIterationNode) n;
- final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
-
- // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
- // One check is for free during the translation, we do the other check here as a pre-condition
- {
- StepFunctionValidator wsf = new StepFunctionValidator();
- iter.getNextWorkset().accept(wsf);
- if (!wsf.foundWorkset) {
- throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
- "This is a prerequisite in delta iterations.");
- }
- }
-
- // calculate the closure of the anonymous function
- HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
-
- // first, recursively build the data flow for the step function
- final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
- this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure);
-
- // descend from the solution set delta. check that it depends on both the workset
- // and the solution set. If it does depend on both, this descend should create both nodes
- iter.getSolutionSetDelta().accept(recursiveCreator);
-
- final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
-
- if (worksetNode == null) {
- throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
- "This is a prerequisite in delta iterations.");
- }
-
- iter.getNextWorkset().accept(recursiveCreator);
-
- SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
-
- if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
- solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
- }
- else {
- for (DagConnection conn : solutionSetNode.getOutgoingConnections()) {
- OptimizerNode successor = conn.getTarget();
-
- if (successor.getClass() == JoinNode.class) {
- // find out which input to the match the solution set is
- JoinNode mn = (JoinNode) successor;
- if (mn.getFirstPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(0);
- } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
- mn.makeJoinWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else if (successor.getClass() == CoGroupNode.class) {
- CoGroupNode cg = (CoGroupNode) successor;
- if (cg.getFirstPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(0);
- } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
- cg.makeCoGroupWithSolutionSet(1);
- } else {
- throw new CompilerException();
- }
- }
- else {
- throw new InvalidProgramException(
- "Error: The only operations allowed on the solution set are Join and CoGroup.");
- }
- }
- }
-
- final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
- final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
-
- // set the step function nodes to the iteration node
- iterNode.setPartialSolution(solutionSetNode, worksetNode);
- iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
-
- // go over the contained data flow and mark the dynamic path nodes
- StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
- iterNode.acceptForStepFunction(pathIdentifier);
- }
- }
- }
-
- private static final class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
-
- private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
-
- private final int costWeight;
-
- private StaticDynamicPathIdentifier(int costWeight) {
- this.costWeight = costWeight;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return this.seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- visitable.identifyDynamicPath(this.costWeight);
-
- // check that there is no nested iteration on the dynamic path
- if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
- throw new CompilerException("Nested iterations are currently not supported.");
- }
- }
- }
-
- /**
- * Simple visitor that sets the minimal guaranteed memory per task based on the amount of available memory,
- * the number of memory consumers, and on the task's degree of parallelism.
- */
- public static final class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
-
- private final DataStatistics statistics;
-
- private int id = 1;
-
- public IdAndEstimatesVisitor(DataStatistics statistics) {
- this.statistics = statistics;
- }
-
- @Override
- public boolean preVisit(OptimizerNode visitable) {
- return visitable.getId() == -1;
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {
- // the node ids
- visitable.initId(this.id++);
-
- // connections need to figure out their maximum path depths
- for (DagConnection conn : visitable.getIncomingConnections()) {
- conn.initMaxDepth();
- }
- for (DagConnection conn : visitable.getBroadcastConnections()) {
- conn.initMaxDepth();
- }
-
- // the estimates
- visitable.computeOutputEstimates(this.statistics);
-
- // if required, recurse into the step function
- if (visitable instanceof IterationNode) {
- ((IterationNode) visitable).acceptForStepFunction(this);
- }
- }
- }
-
- /**
- * Visitor that computes the interesting properties for each node in the plan. On its recursive
- * depth-first descend, it propagates all interesting properties top-down.
- */
- public static final class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
-
- private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
-
- /**
- * Creates a new visitor that computes the interesting properties for all nodes in the plan.
- * It uses the given cost estimator used to compute the maximal costs for an interesting property.
- *
- * @param estimator
- * The cost estimator to estimate the maximal costs for interesting properties.
- */
- public InterestingPropertyVisitor(CostEstimator estimator) {
- this.estimator = estimator;
- }
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- // The interesting properties must be computed on the descend. In case a node has multiple outputs,
- // that computation must happen during the last descend.
-
- if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
- node.computeUnionOfInterestingPropertiesFromSuccessors();
- node.computeInterestingPropertiesForInputs(this.estimator);
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(OptimizerNode visitable) {}
- }
-
- /**
- * On its re-ascend (post visit) this visitor, computes auxiliary maps that are needed to support plans
- * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
- * output into more than one other node).
- */
- public static final class BranchesVisitor implements Visitor<OptimizerNode> {
-
- @Override
- public boolean preVisit(OptimizerNode node) {
- return node.getOpenBranches() == null;
- }
-
- @Override
- public void postVisit(OptimizerNode node) {
- if (node instanceof IterationNode) {
- ((IterationNode) node).acceptForStepFunction(this);
- }
-
- node.computeUnclosedBranchStack();
- }
- }
-
- /**
- * Finalization of the plan:
- * - The graph of nodes is double-linked (links from child to parent are inserted)
- * - If unions join static and dynamic paths, the cache is marked as a memory consumer
- * - Relative memory fractions are assigned to all nodes.
- * - All nodes are collected into a set.
- */
- private static final class PlanFinalizer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
-
- private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
-
- private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
-
- private final Deque<IterationPlanNode> stackOfIterationNodes;
-
- private int memoryConsumerWeights; // a counter of all memory consumers
-
- /**
- * Creates a new plan finalizer.
- */
- private PlanFinalizer() {
- this.allNodes = new HashSet<PlanNode>();
- this.sources = new ArrayList<SourcePlanNode>();
- this.sinks = new ArrayList<SinkPlanNode>();
- this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
- }
-
- private OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
- this.memoryConsumerWeights = 0;
-
- // traverse the graph
- for (SinkPlanNode node : sinks) {
- node.accept(this);
- }
-
- // assign the memory to each node
- if (this.memoryConsumerWeights > 0) {
- for (PlanNode node : this.allNodes) {
- // assign memory to the driver strategy of the node
- final int consumerWeight = node.getMemoryConsumerWeight();
- if (consumerWeight > 0) {
- final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
- node.setRelativeMemoryPerSubtask(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
- node.getProgramOperator().getName() + ".");
- }
- }
-
- // assign memory to the local and global strategies of the channels
- for (Channel c : node.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- final double relativeMem = 1.0 / this.memoryConsumerWeights;
- c.setRelativeMemoryLocalStrategy(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
- "instance of " + c + ".");
- }
- }
- if (c.getTempMode() != TempMode.NONE) {
- final double relativeMem = 1.0/ this.memoryConsumerWeights;
- c.setRelativeTempMemory(relativeMem);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
- "table for " + c + ".");
- }
- }
- }
- }
- }
- return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
- }
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- // if we come here again, prevent a further descend
- if (!this.allNodes.add(visitable)) {
- return false;
- }
-
- if (visitable instanceof SinkPlanNode) {
- this.sinks.add((SinkPlanNode) visitable);
- }
- else if (visitable instanceof SourcePlanNode) {
- this.sources.add((SourcePlanNode) visitable);
- }
- else if (visitable instanceof BinaryUnionPlanNode) {
- BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- if (unionNode.unionsStaticAndDynamicPath()) {
- unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
- }
- }
- else if (visitable instanceof BulkPartialSolutionPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
- }
- else if (visitable instanceof WorksetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
- else if (visitable instanceof SolutionSetPlanNode) {
- // tell the partial solution about the iteration node that contains it
- final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
- final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
-
- // sanity check!
- if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
- throw new CompilerException("Bug: Error finalizing the plan. " +
- "Cannot associate the node for a partial solutions with its containing iteration.");
- }
- sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
- }
-
- // double-connect the connections. previously, only parents knew their children, because
- // one child candidate could have been referenced by multiple parents.
- for (Channel conn : visitable.getInputs()) {
- conn.setTarget(visitable);
- conn.getSource().addOutgoingChannel(conn);
- }
-
- for (Channel c : visitable.getBroadcastInputs()) {
- c.setTarget(visitable);
- c.getSource().addOutgoingChannel(c);
- }
-
- // count the memory consumption
- this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
- for (Channel c : visitable.getInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
- for (Channel c : visitable.getBroadcastInputs()) {
- if (c.getLocalStrategy().dams()) {
- this.memoryConsumerWeights++;
- }
- if (c.getTempMode() != TempMode.NONE) {
- this.memoryConsumerWeights++;
- }
- }
-
- // pass the visitor to the iteraton's step function
- if (visitable instanceof IterationPlanNode) {
- // push the iteration node onto the stack
- final IterationPlanNode iterNode = (IterationPlanNode) visitable;
- this.stackOfIterationNodes.addLast(iterNode);
-
- // recurse
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
-
- // pop the iteration node from the stack
- this.stackOfIterationNodes.removeLast();
- }
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {}
- }
-
- /**
- * A visitor that traverses the graph and collects cascading binary unions into a single n-ary
- * union operator. The exception is, when on of the union inputs is materialized, such as in the
- * static-code-path-cache in iterations.
- */
- private static final class BinaryUnionReplacer implements Visitor<PlanNode> {
-
- private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
-
- @Override
- public boolean preVisit(PlanNode visitable) {
- if (this.seenBefore.add(visitable)) {
- if (visitable instanceof IterationPlanNode) {
- ((IterationPlanNode) visitable).acceptForStepFunction(this);
- }
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
-
- if (visitable instanceof BinaryUnionPlanNode) {
-
- final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
- final Channel in1 = unionNode.getInput1();
- final Channel in2 = unionNode.getInput2();
-
- if (!unionNode.unionsStaticAndDynamicPath()) {
-
- // both on static path, or both on dynamic path. we can collapse them
- NAryUnionPlanNode newUnionNode;
-
- List<Channel> inputs = new ArrayList<Channel>();
- collect(in1, inputs);
- collect(in2, inputs);
-
- newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
- unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
-
- newUnionNode.setParallelism(unionNode.getParallelism());
-
- for (Channel c : inputs) {
- c.setTarget(newUnionNode);
- }
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- else {
- // union between the static and the dynamic path. we need to handle this for now
- // through a special union operator
-
- // make sure that the first input is the cached (static) and the second input is the dynamic
- if (in1.isOnDynamicPath()) {
- BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
-
- in1.setTarget(newUnionNode);
- in2.setTarget(newUnionNode);
-
- for (Channel channel : unionNode.getOutgoingChannels()) {
- channel.swapUnionNodes(newUnionNode);
- newUnionNode.addOutgoingChannel(channel);
- }
- }
- }
- }
- }
-
- private void collect(Channel in, List<Channel> inputs) {
- if (in.getSource() instanceof NAryUnionPlanNode) {
- // sanity check
- if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
- throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
- }
- if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
- throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
- }
-
- inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
- } else {
- // is not a collapsed union node, so we take the channel directly
- inputs.add(in);
- }
- }
- }
-
- private static final class StepFunctionValidator implements Visitor<Operator<?>> {
-
- private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
-
- private boolean foundWorkset;
-
- @Override
- public boolean preVisit(Operator<?> visitable) {
- if (visitable instanceof WorksetPlaceHolder) {
- foundWorkset = true;
- }
-
- return (!foundWorkset) && seenBefore.add(visitable);
- }
-
- @Override
- public void postVisit(Operator<?> visitable) {}
- }
// ------------------------------------------------------------------------
// Miscellaneous
@@ -1363,10 +560,12 @@ public class Optimizer {
throw rtex;
}
}
- } catch (ClassNotFoundException cnfex) {
+ }
+ catch (ClassNotFoundException cnfex) {
throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex);
- } catch (ClassCastException ccex) {
- throw new CompilerException("Class '" + className + "' is not an optimizer post passer.", ccex);
+ }
+ catch (ClassCastException ccex) {
+ throw new CompilerException("Class '" + className + "' is not an optimizer post-pass.", ccex);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
index c55d17a..55b8785 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/BulkIterationNode.java
@@ -31,7 +31,7 @@ import org.apache.flink.api.common.operators.base.BulkIterationBase;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dag.WorksetIterationNode.SingleRootJoiner;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
index 360f579..4e65976 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/DagConnection.java
@@ -79,7 +79,7 @@ public class DagConnection implements EstimateProvider, DumpableConnection<Optim
* The data exchange mode (pipelined / batch / batch only for shuffles / ... )
*/
public DagConnection(OptimizerNode source, OptimizerNode target,
- ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
+ ShipStrategyType shipStrategy, ExecutionMode exchangeMode)
{
if (source == null || target == null) {
throw new NullPointerException("Source and target must not be null.");
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
index 71a49f3..e85f289 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
@@ -32,7 +32,7 @@ import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
import org.apache.flink.optimizer.CompilerException;
import org.apache.flink.optimizer.DataStatistics;
-import org.apache.flink.optimizer.Optimizer.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
import org.apache.flink.optimizer.costs.CostEstimator;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.InterestingProperties;
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
new file mode 100644
index 0000000..bd35b5c
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BinaryUnionReplacer.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A traversal that collects cascading binary unions into a single n-ary
+ * union operator. The exception is, when on of the union inputs is materialized, such as in the
+ * static-code-path-cache in iterations.
+ */
+public class BinaryUnionReplacer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (this.seenBefore.add(visitable)) {
+ if (visitable instanceof IterationPlanNode) {
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {
+
+ if (visitable instanceof BinaryUnionPlanNode) {
+
+ final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ final Channel in1 = unionNode.getInput1();
+ final Channel in2 = unionNode.getInput2();
+
+ if (!unionNode.unionsStaticAndDynamicPath()) {
+
+ // both on static path, or both on dynamic path. we can collapse them
+ NAryUnionPlanNode newUnionNode;
+
+ List<Channel> inputs = new ArrayList<Channel>();
+ collect(in1, inputs);
+ collect(in2, inputs);
+
+ newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs,
+ unionNode.getGlobalProperties(), unionNode.getCumulativeCosts());
+
+ newUnionNode.setParallelism(unionNode.getParallelism());
+
+ for (Channel c : inputs) {
+ c.setTarget(newUnionNode);
+ }
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ else {
+ // union between the static and the dynamic path. we need to handle this for now
+ // through a special union operator
+
+ // make sure that the first input is the cached (static) and the second input is the dynamic
+ if (in1.isOnDynamicPath()) {
+ BinaryUnionPlanNode newUnionNode = new BinaryUnionPlanNode(unionNode);
+
+ in1.setTarget(newUnionNode);
+ in2.setTarget(newUnionNode);
+
+ for (Channel channel : unionNode.getOutgoingChannels()) {
+ channel.swapUnionNodes(newUnionNode);
+ newUnionNode.addOutgoingChannel(channel);
+ }
+ }
+ }
+ }
+ }
+
+ public void collect(Channel in, List<Channel> inputs) {
+ if (in.getSource() instanceof NAryUnionPlanNode) {
+ // sanity check
+ if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
+ }
+ if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
+ throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
+ }
+
+ inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
+ } else {
+ // is not a collapsed union node, so we take the channel directly
+ inputs.add(in);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
new file mode 100644
index 0000000..4730546
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/BranchesVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * This traversal of the optimizer DAG computes the information needed to track
+ * branches and joins in the data flow. This is important to support plans
+ * that are not a minimally connected DAG (Such plans are not trees, but at least one node feeds its
+ * output into more than one other node).
+ */
+public final class BranchesVisitor implements Visitor<OptimizerNode> {
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ return node.getOpenBranches() == null;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode node) {
+ if (node instanceof IterationNode) {
+ ((IterationNode) node).acceptForStepFunction(this);
+ }
+
+ node.computeUnclosedBranchStack();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
new file mode 100644
index 0000000..160ef95
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/GraphCreatingVisitor.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.operators.GenericDataSinkBase;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Union;
+import org.apache.flink.api.common.operators.base.BulkIterationBase;
+import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
+import org.apache.flink.api.common.operators.base.CrossOperatorBase;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.api.common.operators.base.FilterOperatorBase;
+import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupCombineOperatorBase;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.common.operators.base.MapPartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dag.BulkIterationNode;
+import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
+import org.apache.flink.optimizer.dag.CoGroupNode;
+import org.apache.flink.optimizer.dag.CollectorMapNode;
+import org.apache.flink.optimizer.dag.CrossNode;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dag.FilterNode;
+import org.apache.flink.optimizer.dag.FlatMapNode;
+import org.apache.flink.optimizer.dag.GroupCombineNode;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.JoinNode;
+import org.apache.flink.optimizer.dag.MapNode;
+import org.apache.flink.optimizer.dag.MapPartitionNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.PartitionNode;
+import org.apache.flink.optimizer.dag.ReduceNode;
+import org.apache.flink.optimizer.dag.SolutionSetNode;
+import org.apache.flink.optimizer.dag.SortPartitionNode;
+import org.apache.flink.optimizer.dag.WorksetIterationNode;
+import org.apache.flink.optimizer.dag.WorksetNode;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This traversal creates the optimizer DAG from a program.
+ * It works as a visitor that walks the program's flow in a depth-first fashion, starting from the data sinks.
+ * During the descend, it creates an optimizer node for each operator, respectively data source or -sink.
+ * During the ascend, it connects the nodes to the full graph.
+ */
+public class GraphCreatingVisitor implements Visitor<Operator<?>> {
+
+ private final Map<Operator<?>, OptimizerNode> con2node; // map from the operator objects to their
+ // corresponding optimizer nodes
+
+ private final List<DataSinkNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final int defaultParallelism; // the default degree of parallelism
+
+ private final GraphCreatingVisitor parent; // reference to enclosing creator, in case of a recursive translation
+
+ private final ExecutionMode defaultDataExchangeMode;
+
+ private final boolean forceDOP;
+
+
+ public GraphCreatingVisitor(int defaultParallelism, ExecutionMode defaultDataExchangeMode) {
+ this(null, false, defaultParallelism, defaultDataExchangeMode, null);
+ }
+
+ private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceDOP, int defaultParallelism,
+ ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
+ if (closure == null){
+ con2node = new HashMap<Operator<?>, OptimizerNode>();
+ } else {
+ con2node = closure;
+ }
+
+ this.sinks = new ArrayList<DataSinkNode>(2);
+ this.defaultParallelism = defaultParallelism;
+ this.parent = parent;
+ this.defaultDataExchangeMode = dataExchangeMode;
+ this.forceDOP = forceDOP;
+ }
+
+ public List<DataSinkNode> getSinks() {
+ return sinks;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public boolean preVisit(Operator<?> c) {
+ // check if we have been here before
+ if (this.con2node.containsKey(c)) {
+ return false;
+ }
+
+ final OptimizerNode n;
+
+ // create a node for the operator (or sink or source) if we have not been here before
+ if (c instanceof GenericDataSinkBase) {
+ DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
+ this.sinks.add(dsn);
+ n = dsn;
+ }
+ else if (c instanceof GenericDataSourceBase) {
+ n = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
+ }
+ else if (c instanceof MapOperatorBase) {
+ n = new MapNode((MapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof MapPartitionOperatorBase) {
+ n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
+ n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FlatMapOperatorBase) {
+ n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof FilterOperatorBase) {
+ n = new FilterNode((FilterOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof ReduceOperatorBase) {
+ n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
+ }
+ else if (c instanceof GroupCombineOperatorBase) {
+ n = new GroupCombineNode((GroupCombineOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof GroupReduceOperatorBase) {
+ n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
+ }
+ else if (c instanceof JoinOperatorBase) {
+ n = new JoinNode((JoinOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CoGroupOperatorBase) {
+ n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof CrossOperatorBase) {
+ n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
+ }
+ else if (c instanceof BulkIterationBase) {
+ n = new BulkIterationNode((BulkIterationBase<?>) c);
+ }
+ else if (c instanceof DeltaIterationBase) {
+ n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
+ }
+ else if (c instanceof Union){
+ n = new BinaryUnionNode((Union<?>) c);
+ }
+ else if (c instanceof PartitionOperatorBase) {
+ n = new PartitionNode((PartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof SortPartitionOperatorBase) {
+ n = new SortPartitionNode((SortPartitionOperatorBase<?>) c);
+ }
+ else if (c instanceof BulkIterationBase.PartialSolutionPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final BulkIterationBase.PartialSolutionPlaceHolder<?> holder = (BulkIterationBase.PartialSolutionPlaceHolder<?>) c;
+ final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
+ final BulkIterationNode containingIterationNode =
+ (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof DeltaIterationBase.WorksetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final DeltaIterationBase.WorksetPlaceHolder<?> holder = (DeltaIterationBase.WorksetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ WorksetNode p = new WorksetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else if (c instanceof DeltaIterationBase.SolutionSetPlaceHolder) {
+ if (this.parent == null) {
+ throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
+ }
+
+ final DeltaIterationBase.SolutionSetPlaceHolder<?> holder = (DeltaIterationBase.SolutionSetPlaceHolder<?>) c;
+ final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
+ final WorksetIterationNode containingIterationNode =
+ (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
+
+ // catch this for the recursive translation of step functions
+ SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
+ p.setDegreeOfParallelism(containingIterationNode.getParallelism());
+ n = p;
+ }
+ else {
+ throw new IllegalArgumentException("Unknown operator type: " + c);
+ }
+
+ this.con2node.put(c, n);
+
+ // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
+ // key-less reducer (all-reduce)
+ if (n.getParallelism() < 1) {
+ // set the degree of parallelism
+ int par = c.getDegreeOfParallelism();
+ if (par > 0) {
+ if (this.forceDOP && par != this.defaultParallelism) {
+ par = this.defaultParallelism;
+ Optimizer.LOG.warn("The parallelism of nested dataflows (such as step functions in iterations) is " +
+ "currently fixed to the parallelism of the surrounding operator (the iteration).");
+ }
+ } else {
+ par = this.defaultParallelism;
+ }
+ n.setDegreeOfParallelism(par);
+ }
+
+ return true;
+ }
+
+ @Override
+ public void postVisit(Operator<?> c) {
+
+ OptimizerNode n = this.con2node.get(c);
+
+ // first connect to the predecessors
+ n.setInput(this.con2node, this.defaultDataExchangeMode);
+ n.setBroadcastInputs(this.con2node, this.defaultDataExchangeMode);
+
+ // if the node represents a bulk iteration, we recursively translate the data flow now
+ if (n instanceof BulkIterationNode) {
+ final BulkIterationNode iterNode = (BulkIterationNode) n;
+ final BulkIterationBase<?> iter = iterNode.getIterationContract();
+
+ // pass a copy of the no iterative part into the iteration translation,
+ // in case the iteration references its closure
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
+ iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ BulkPartialSolutionNode partialSolution;
+
+ iter.getNextPartialSolution().accept(recursiveCreator);
+
+ partialSolution = (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
+ OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
+ if (partialSolution == null) {
+ throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
+ }
+
+
+ OptimizerNode terminationCriterion = null;
+
+ if (iter.getTerminationCriterion() != null) {
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+
+ // no intermediate node yet, traverse from the termination criterion to build the missing parts
+ if (terminationCriterion == null) {
+ iter.getTerminationCriterion().accept(recursiveCreator);
+ terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
+ }
+ }
+
+ iterNode.setPartialSolution(partialSolution);
+ iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(identifier);
+ }
+ else if (n instanceof WorksetIterationNode) {
+ final WorksetIterationNode iterNode = (WorksetIterationNode) n;
+ final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();
+
+ // we need to ensure that both the next-workset and the solution-set-delta depend on the workset.
+ // One check is for free during the translation, we do the other check here as a pre-condition
+ {
+ StepFunctionValidator wsf = new StepFunctionValidator();
+ iter.getNextWorkset().accept(wsf);
+ if (!wsf.hasFoundWorkset()) {
+ throw new CompilerException("In the given program, the next workset does not depend on the workset. " +
+ "This is a prerequisite in delta iterations.");
+ }
+ }
+
+ // calculate the closure of the anonymous function
+ HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);
+
+ // first, recursively build the data flow for the step function
+ final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(
+ this, true, iterNode.getParallelism(), defaultDataExchangeMode, closure);
+
+ // descend from the solution set delta. check that it depends on both the workset
+ // and the solution set. If it does depend on both, this descend should create both nodes
+ iter.getSolutionSetDelta().accept(recursiveCreator);
+
+ final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
+
+ if (worksetNode == null) {
+ throw new CompilerException("In the given program, the solution set delta does not depend on the workset." +
+ "This is a prerequisite in delta iterations.");
+ }
+
+ iter.getNextWorkset().accept(recursiveCreator);
+
+ SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
+
+ if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
+ solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
+ }
+ else {
+ for (DagConnection conn : solutionSetNode.getOutgoingConnections()) {
+ OptimizerNode successor = conn.getTarget();
+
+ if (successor.getClass() == JoinNode.class) {
+ // find out which input to the match the solution set is
+ JoinNode mn = (JoinNode) successor;
+ if (mn.getFirstPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(0);
+ } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
+ mn.makeJoinWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else if (successor.getClass() == CoGroupNode.class) {
+ CoGroupNode cg = (CoGroupNode) successor;
+ if (cg.getFirstPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(0);
+ } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
+ cg.makeCoGroupWithSolutionSet(1);
+ } else {
+ throw new CompilerException();
+ }
+ }
+ else {
+ throw new InvalidProgramException(
+ "Error: The only operations allowed on the solution set are Join and CoGroup.");
+ }
+ }
+ }
+
+ final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
+ final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
+
+ // set the step function nodes to the iteration node
+ iterNode.setPartialSolution(solutionSetNode, worksetNode);
+ iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode, defaultDataExchangeMode);
+
+ // go over the contained data flow and mark the dynamic path nodes
+ StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
+ iterNode.acceptForStepFunction(pathIdentifier);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
new file mode 100644
index 0000000..b5c09e5
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/IdAndEstimatesVisitor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dag.DagConnection;
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * This traversal of the optimizer DAG assigns IDs to each node (in a pre-order fashion),
+ * and calls each node to compute its estimates. The latter happens in the postVisit function,
+ * where it is guaranteed that all predecessors have computed their estimates.
+ */
+public class IdAndEstimatesVisitor implements Visitor<OptimizerNode> {
+
+ private final DataStatistics statistics;
+
+ private int id = 1;
+
+ public IdAndEstimatesVisitor(DataStatistics statistics) {
+ this.statistics = statistics;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return visitable.getId() == -1;
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ // the node ids
+ visitable.initId(this.id++);
+
+ // connections need to figure out their maximum path depths
+ for (DagConnection conn : visitable.getIncomingConnections()) {
+ conn.initMaxDepth();
+ }
+ for (DagConnection conn : visitable.getBroadcastConnections()) {
+ conn.initMaxDepth();
+ }
+
+ // the estimates
+ visitable.computeOutputEstimates(this.statistics);
+
+ // if required, recurse into the step function
+ if (visitable instanceof IterationNode) {
+ ((IterationNode) visitable).acceptForStepFunction(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
new file mode 100644
index 0000000..c2e81a8
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Visitor that computes the interesting properties for each node in the optimizer DAG. On its recursive
+ * depth-first descend, it propagates all interesting properties top-down.
+ */
+public class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
+
+ private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
+
+ /**
+ * Creates a new visitor that computes the interesting properties for all nodes in the plan.
+ * It uses the given cost estimator used to compute the maximal costs for an interesting property.
+ *
+ * @param estimator
+ * The cost estimator to estimate the maximal costs for interesting properties.
+ */
+ public InterestingPropertyVisitor(CostEstimator estimator) {
+ this.estimator = estimator;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ // The interesting properties must be computed on the descend. In case a node has multiple outputs,
+ // that computation must happen during the last descend.
+
+ if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
+ node.computeUnionOfInterestingPropertiesFromSuccessors();
+ node.computeInterestingPropertiesForInputs(this.estimator);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
new file mode 100644
index 0000000..58aa3c1
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This visitor traverses the selected execution plan and finalizes it:
+ *
+ * <ul>
+ * <li>The graph of nodes is double-linked (links from child to parent are inserted).</li>
+ * <li>If unions join static and dynamic paths, the cache is marked as a memory consumer.</li>
+ * <li>Relative memory fractions are assigned to all nodes.</li>
+ * <li>All nodes are collected into a set.</li>
+ * </ul>
+ */
+public class PlanFinalizer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
+
+ private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
+
+ private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final Deque<IterationPlanNode> stackOfIterationNodes;
+
+ private int memoryConsumerWeights; // a counter of all memory consumers
+
+ /**
+ * Creates a new plan finalizer.
+ */
+ public PlanFinalizer() {
+ this.allNodes = new HashSet<PlanNode>();
+ this.sources = new ArrayList<SourcePlanNode>();
+ this.sinks = new ArrayList<SinkPlanNode>();
+ this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
+ }
+
+ public OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
+ this.memoryConsumerWeights = 0;
+
+ // traverse the graph
+ for (SinkPlanNode node : sinks) {
+ node.accept(this);
+ }
+
+ // assign the memory to each node
+ if (this.memoryConsumerWeights > 0) {
+ for (PlanNode node : this.allNodes) {
+ // assign memory to the driver strategy of the node
+ final int consumerWeight = node.getMemoryConsumerWeight();
+ if (consumerWeight > 0) {
+ final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
+ node.setRelativeMemoryPerSubtask(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
+ node.getProgramOperator().getName() + ".");
+ }
+ }
+
+ // assign memory to the local and global strategies of the channels
+ for (Channel c : node.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ final double relativeMem = 1.0 / this.memoryConsumerWeights;
+ c.setRelativeMemoryLocalStrategy(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
+ "instance of " + c + ".");
+ }
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ final double relativeMem = 1.0/ this.memoryConsumerWeights;
+ c.setRelativeTempMemory(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
+ "table for " + c + ".");
+ }
+ }
+ }
+ }
+ }
+ return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
+ }
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ // if we come here again, prevent a further descend
+ if (!this.allNodes.add(visitable)) {
+ return false;
+ }
+
+ if (visitable instanceof SinkPlanNode) {
+ this.sinks.add((SinkPlanNode) visitable);
+ }
+ else if (visitable instanceof SourcePlanNode) {
+ this.sources.add((SourcePlanNode) visitable);
+ }
+ else if (visitable instanceof BinaryUnionPlanNode) {
+ BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ if (unionNode.unionsStaticAndDynamicPath()) {
+ unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
+ }
+ }
+ else if (visitable instanceof BulkPartialSolutionPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof WorksetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof SolutionSetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+
+ // double-connect the connections. previously, only parents knew their children, because
+ // one child candidate could have been referenced by multiple parents.
+ for (Channel conn : visitable.getInputs()) {
+ conn.setTarget(visitable);
+ conn.getSource().addOutgoingChannel(conn);
+ }
+
+ for (Channel c : visitable.getBroadcastInputs()) {
+ c.setTarget(visitable);
+ c.getSource().addOutgoingChannel(c);
+ }
+
+ // count the memory consumption
+ this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
+ for (Channel c : visitable.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+ for (Channel c : visitable.getBroadcastInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+
+ // pass the visitor to the iteraton's step function
+ if (visitable instanceof IterationPlanNode) {
+ // push the iteration node onto the stack
+ final IterationPlanNode iterNode = (IterationPlanNode) visitable;
+ this.stackOfIterationNodes.addLast(iterNode);
+
+ // recurse
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+
+ // pop the iteration node from the stack
+ this.stackOfIterationNodes.removeLast();
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
new file mode 100644
index 0000000..c0dc4dd
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A traversal that goes over the program data flow of an iteration and makes the nodes
+ * that depend on the partial solution (the data set recomputed in each iteration) as "dynamic"
+ * and the other nodes as "static".
+ */
+public class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
+
+ private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
+
+ private final int costWeight;
+
+ public StaticDynamicPathIdentifier(int costWeight) {
+ this.costWeight = costWeight;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return this.seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ visitable.identifyDynamicPath(this.costWeight);
+
+ // check that there is no nested iteration on the dynamic path
+ if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
+ throw new CompilerException("Nested iterations are currently not supported.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/5757850c/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
new file mode 100644
index 0000000..d359490
--- /dev/null
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.util.Visitor;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A traversal that checks if the Workset of a delta iteration is used in the data flow
+ * of its step function.
+ */
+public class StepFunctionValidator implements Visitor<Operator<?>> {
+
+ private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
+
+ private boolean foundWorkset;
+
+ @Override
+ public boolean preVisit(Operator<?> visitable) {
+ if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
+ foundWorkset = true;
+ }
+
+ return (!foundWorkset) && seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(Operator<?> visitable) {}
+
+ public boolean hasFoundWorkset() {
+ return foundWorkset;
+ }
+}
[13/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
new file mode 100644
index 0000000..df05b64
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkPartialSolutionPlanNode.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.BulkPartialSolutionNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Plan candidate node for partial solution of a bulk iteration.
+ */
+public class BulkPartialSolutionPlanNode extends PlanNode {
+
+ private static final Costs NO_COSTS = new Costs();
+
+ private BulkIterationPlanNode containingIterationNode;
+
+ private Channel initialInput;
+
+ public Object postPassHelper;
+
+
+ public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
+ GlobalProperties gProps, LocalProperties lProps,
+ Channel initialInput)
+ {
+ super(template, nodeName, DriverStrategy.NONE);
+
+ this.globalProps = gProps;
+ this.localProps = lProps;
+ this.initialInput = initialInput;
+
+ // the partial solution does not cost anything
+ this.nodeCosts = NO_COSTS;
+ this.cumulativeCosts = NO_COSTS;
+
+ if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
+ }
+
+ this.branchPlan.putAll(initialInput.getSource().branchPlan);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public BulkPartialSolutionNode getPartialSolutionNode() {
+ return (BulkPartialSolutionNode) this.template;
+ }
+
+ public BulkIterationPlanNode getContainingIterationNode() {
+ return this.containingIterationNode;
+ }
+
+ public void setContainingIterationNode(BulkIterationPlanNode containingIterationNode) {
+ this.containingIterationNode = containingIterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ return Collections.<PlanNode>emptyList();
+ }
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.<Channel>emptyList();
+ }
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+ SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
+ if (res == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (res == FOUND_SOURCE) {
+ return (this.initialInput.getLocalStrategy().dams() ||
+ this.initialInput.getTempMode().breaksPipeline() ||
+ getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
+ FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
+ }
+ else {
+ return NOT_FOUND;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/Channel.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/Channel.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/Channel.java
new file mode 100644
index 0000000..875d1c3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/Channel.java
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.EstimateProvider;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plandump.DumpableConnection;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+
+/**
+ * A Channel represents the result produced by an operator and the data exchange
+ * before the consumption by the target operator.
+ *
+ * The channel defines and tracks various properties and characteristics of the
+ * data set and data exchange.
+ *
+ * Data set characteristics:
+ * <ul>
+ * <li>The "global properties" of the data, i.e., how the data is distributed across
+ * partitions</li>
+ * <li>The "required global properties" of the data, i.e., the global properties that, if absent,
+ * would cause the program to return a wrong result.</li>
+ * <li>The "local properties" of the data, i.e., how the data is organized within a partition</li>
+ * <li>The "required local properties" of the data, i.e., the local properties that, if absent,
+ * would cause the program to return a wrong result.</li>
+ * </ul>
+ *
+ * Data exchange parameters:
+ * <ul>
+ * <li>The "ship strategy", i.e., whether to forward the data, shuffle it, broadcast it, ...</li>
+ * <li>The "ship keys", which are the positions of the key fields in the exchanged records.</li>
+ * <li>The "data exchange mode", which defines whether to pipeline or batch the exchange</li>
+ * <li>Several more...</li>
+ * </ul>
+ */
+public class Channel implements EstimateProvider, Cloneable, DumpableConnection<PlanNode> {
+
+ private PlanNode source;
+
+ private PlanNode target;
+
+ private ShipStrategyType shipStrategy = ShipStrategyType.NONE;
+
+ private DataExchangeMode dataExchangeMode;
+
+ private LocalStrategy localStrategy = LocalStrategy.NONE;
+
+ private FieldList shipKeys;
+
+ private FieldList localKeys;
+
+ private boolean[] shipSortOrder;
+
+ private boolean[] localSortOrder;
+
+ private RequestedGlobalProperties requiredGlobalProps;
+
+ private RequestedLocalProperties requiredLocalProps;
+
+ private GlobalProperties globalProps;
+
+ private LocalProperties localProps;
+
+ private TypeSerializerFactory<?> serializer;
+
+ private TypeComparatorFactory<?> shipStrategyComparator;
+
+ private TypeComparatorFactory<?> localStrategyComparator;
+
+ private DataDistribution dataDistribution;
+
+ private Partitioner<?> partitioner;
+
+ private TempMode tempMode;
+
+ private double relativeTempMemory;
+
+ private double relativeMemoryLocalStrategy;
+
+ private int replicationFactor = 1;
+
+ // --------------------------------------------------------------------------------------------
+
+ public Channel(PlanNode sourceNode) {
+ this(sourceNode, null);
+ }
+
+ public Channel(PlanNode sourceNode, TempMode tempMode) {
+ this.source = sourceNode;
+ this.tempMode = (tempMode == null ? TempMode.NONE : tempMode);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Accessors
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the source of this Channel.
+ *
+ * @return The source.
+ */
+ @Override
+ public PlanNode getSource() {
+ return this.source;
+ }
+
+ /**
+ * Sets the target of this Channel.
+ *
+ * @param target The target.
+ */
+ public void setTarget(PlanNode target) {
+ this.target = target;
+ }
+
+ /**
+ * Gets the target of this Channel.
+ *
+ * @return The target.
+ */
+ public PlanNode getTarget() {
+ return this.target;
+ }
+
+ public void setShipStrategy(ShipStrategyType strategy, DataExchangeMode dataExchangeMode) {
+ setShipStrategy(strategy, null, null, null, dataExchangeMode);
+ }
+
+ public void setShipStrategy(ShipStrategyType strategy, FieldList keys, DataExchangeMode dataExchangeMode) {
+ setShipStrategy(strategy, keys, null, null, dataExchangeMode);
+ }
+
+ public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
+ boolean[] sortDirection, DataExchangeMode dataExchangeMode) {
+ setShipStrategy(strategy, keys, sortDirection, null, dataExchangeMode);
+ }
+
+ public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
+ Partitioner<?> partitioner, DataExchangeMode dataExchangeMode) {
+ setShipStrategy(strategy, keys, null, partitioner, dataExchangeMode);
+ }
+
+ public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
+ boolean[] sortDirection, Partitioner<?> partitioner,
+ DataExchangeMode dataExchangeMode) {
+ this.shipStrategy = strategy;
+ this.shipKeys = keys;
+ this.shipSortOrder = sortDirection;
+ this.partitioner = partitioner;
+ this.dataExchangeMode = dataExchangeMode;
+ this.globalProps = null; // reset the global properties
+ }
+
+ /**
+ * Gets the data exchange mode (batch / streaming) to use for the data
+ * exchange of this channel.
+ *
+ * @return The data exchange mode of this channel.
+ */
+ public DataExchangeMode getDataExchangeMode() {
+ return dataExchangeMode;
+ }
+
+ public ShipStrategyType getShipStrategy() {
+ return this.shipStrategy;
+ }
+
+ public FieldList getShipStrategyKeys() {
+ return this.shipKeys;
+ }
+
+ public boolean[] getShipStrategySortOrder() {
+ return this.shipSortOrder;
+ }
+
+ public void setLocalStrategy(LocalStrategy strategy) {
+ setLocalStrategy(strategy, null, null);
+ }
+
+ public void setLocalStrategy(LocalStrategy strategy, FieldList keys, boolean[] sortDirection) {
+ this.localStrategy = strategy;
+ this.localKeys = keys;
+ this.localSortOrder = sortDirection;
+ this.localProps = null; // reset the local properties
+ }
+
+ public LocalStrategy getLocalStrategy() {
+ return this.localStrategy;
+ }
+
+ public FieldList getLocalStrategyKeys() {
+ return this.localKeys;
+ }
+
+ public boolean[] getLocalStrategySortOrder() {
+ return this.localSortOrder;
+ }
+
+ public void setDataDistribution(DataDistribution dataDistribution) {
+ this.dataDistribution = dataDistribution;
+ }
+
+ public DataDistribution getDataDistribution() {
+ return this.dataDistribution;
+ }
+
+ public Partitioner<?> getPartitioner() {
+ return partitioner;
+ }
+
+ public TempMode getTempMode() {
+ return this.tempMode;
+ }
+
+ /**
+ * Sets the temp mode of the connection.
+ *
+ * @param tempMode
+ * The temp mode of the connection.
+ */
+ public void setTempMode(TempMode tempMode) {
+ this.tempMode = tempMode;
+ }
+
+ /**
+ * Gets the memory for materializing the channel's result from this Channel.
+ *
+ * @return The temp memory.
+ */
+ public double getRelativeTempMemory() {
+ return this.relativeTempMemory;
+ }
+
+ /**
+ * Sets the memory for materializing the channel's result from this Channel.
+ *
+ * @param relativeTempMemory The memory for materialization.
+ */
+ public void setRelativeTempMemory(double relativeTempMemory) {
+ this.relativeTempMemory = relativeTempMemory;
+ }
+
+ /**
+ * Sets the replication factor of the connection.
+ *
+ * @param factor The replication factor of the connection.
+ */
+ public void setReplicationFactor(int factor) {
+ this.replicationFactor = factor;
+ }
+
+ /**
+ * Returns the replication factor of the connection.
+ *
+ * @return The replication factor of the connection.
+ */
+ public int getReplicationFactor() {
+ return this.replicationFactor;
+ }
+
+ /**
+ * Gets the serializer from this Channel.
+ *
+ * @return The serializer.
+ */
+ public TypeSerializerFactory<?> getSerializer() {
+ return serializer;
+ }
+
+ /**
+ * Sets the serializer for this Channel.
+ *
+ * @param serializer The serializer to set.
+ */
+ public void setSerializer(TypeSerializerFactory<?> serializer) {
+ this.serializer = serializer;
+ }
+
+ /**
+ * Gets the ship strategy comparator from this Channel.
+ *
+ * @return The ship strategy comparator.
+ */
+ public TypeComparatorFactory<?> getShipStrategyComparator() {
+ return shipStrategyComparator;
+ }
+
+ /**
+ * Sets the ship strategy comparator for this Channel.
+ *
+ * @param shipStrategyComparator The ship strategy comparator to set.
+ */
+ public void setShipStrategyComparator(TypeComparatorFactory<?> shipStrategyComparator) {
+ this.shipStrategyComparator = shipStrategyComparator;
+ }
+
+ /**
+ * Gets the local strategy comparator from this Channel.
+ *
+ * @return The local strategy comparator.
+ */
+ public TypeComparatorFactory<?> getLocalStrategyComparator() {
+ return localStrategyComparator;
+ }
+
+ /**
+ * Sets the local strategy comparator for this Channel.
+ *
+ * @param localStrategyComparator The local strategy comparator to set.
+ */
+ public void setLocalStrategyComparator(TypeComparatorFactory<?> localStrategyComparator) {
+ this.localStrategyComparator = localStrategyComparator;
+ }
+
+ public double getRelativeMemoryLocalStrategy() {
+ return relativeMemoryLocalStrategy;
+ }
+
+ public void setRelativeMemoryLocalStrategy(double relativeMemoryLocalStrategy) {
+ this.relativeMemoryLocalStrategy = relativeMemoryLocalStrategy;
+ }
+
+ public boolean isOnDynamicPath() {
+ return this.source.isOnDynamicPath();
+ }
+
+ public int getCostWeight() {
+ return this.source.getCostWeight();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Statistic Estimates
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public long getEstimatedOutputSize() {
+ long estimate = this.source.template.getEstimatedOutputSize();
+ return estimate < 0 ? estimate : estimate * this.replicationFactor;
+ }
+
+ @Override
+ public long getEstimatedNumRecords() {
+ long estimate = this.source.template.getEstimatedNumRecords();
+ return estimate < 0 ? estimate : estimate * this.replicationFactor;
+ }
+
+ @Override
+ public float getEstimatedAvgWidthPerOutputRecord() {
+ return this.source.template.getEstimatedAvgWidthPerOutputRecord();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Data Property Handling
+ // --------------------------------------------------------------------------------------------
+
+
+ public RequestedGlobalProperties getRequiredGlobalProps() {
+ return requiredGlobalProps;
+ }
+
+ public void setRequiredGlobalProps(RequestedGlobalProperties requiredGlobalProps) {
+ this.requiredGlobalProps = requiredGlobalProps;
+ }
+
+ public RequestedLocalProperties getRequiredLocalProps() {
+ return requiredLocalProps;
+ }
+
+ public void setRequiredLocalProps(RequestedLocalProperties requiredLocalProps) {
+ this.requiredLocalProps = requiredLocalProps;
+ }
+
+ public GlobalProperties getGlobalProperties() {
+ if (this.globalProps == null) {
+ this.globalProps = this.source.getGlobalProperties().clone();
+ switch (this.shipStrategy) {
+ case BROADCAST:
+ this.globalProps.clearUniqueFieldCombinations();
+ this.globalProps.setFullyReplicated();
+ break;
+ case PARTITION_HASH:
+ this.globalProps.setHashPartitioned(this.shipKeys);
+ break;
+ case PARTITION_RANGE:
+ this.globalProps.setRangePartitioned(Utils.createOrdering(this.shipKeys, this.shipSortOrder));
+ break;
+ case FORWARD:
+ break;
+ case PARTITION_RANDOM:
+ this.globalProps.reset();
+ break;
+ case PARTITION_FORCED_REBALANCE:
+ this.globalProps.setForcedRebalanced();
+ break;
+ case PARTITION_CUSTOM:
+ this.globalProps.setCustomPartitioned(this.shipKeys, this.partitioner);
+ break;
+ case NONE:
+ throw new CompilerException("Cannot produce GlobalProperties before ship strategy is set.");
+ }
+ }
+
+ return this.globalProps;
+ }
+
+ public LocalProperties getLocalProperties() {
+ if (this.localProps == null) {
+ computeLocalPropertiesAfterShippingOnly();
+ switch (this.localStrategy) {
+ case NONE:
+ break;
+ case SORT:
+ case COMBININGSORT:
+ this.localProps = LocalProperties.forOrdering(Utils.createOrdering(this.localKeys, this.localSortOrder));
+ break;
+ default:
+ throw new CompilerException("Unsupported local strategy for channel.");
+ }
+ }
+
+ return this.localProps;
+ }
+
+ private void computeLocalPropertiesAfterShippingOnly() {
+ switch (this.shipStrategy) {
+ case BROADCAST:
+ case PARTITION_HASH:
+ case PARTITION_CUSTOM:
+ case PARTITION_RANGE:
+ case PARTITION_RANDOM:
+ case PARTITION_FORCED_REBALANCE:
+ this.localProps = new LocalProperties();
+ break;
+ case FORWARD:
+ this.localProps = this.source.getLocalProperties();
+ break;
+ case NONE:
+ throw new CompilerException("ShipStrategy has not yet been set.");
+ default:
+ throw new CompilerException("Unknown ShipStrategy.");
+ }
+ }
+
+ public void adjustGlobalPropertiesForFullParallelismChange() {
+ if (this.shipStrategy == null || this.shipStrategy == ShipStrategyType.NONE) {
+ throw new IllegalStateException("Cannot adjust channel for degree of parallelism " +
+ "change before the ship strategy is set.");
+ }
+
+ // make sure the properties are acquired
+ if (this.globalProps == null) {
+ getGlobalProperties();
+ }
+
+ // some strategies globally reestablish properties
+ switch (this.shipStrategy) {
+ case FORWARD:
+ throw new CompilerException("Cannot use FORWARD strategy between operations " +
+ "with different number of parallel instances.");
+ case NONE: // excluded by sanity check. left here for verification check completion
+ case BROADCAST:
+ case PARTITION_HASH:
+ case PARTITION_RANGE:
+ case PARTITION_RANDOM:
+ case PARTITION_FORCED_REBALANCE:
+ case PARTITION_CUSTOM:
+ return;
+ }
+ throw new CompilerException("Unrecognized Ship Strategy Type: " + this.shipStrategy);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Utility method used while swapping binary union nodes for n-ary union nodes.
+ */
+ public void swapUnionNodes(PlanNode newUnionNode) {
+ if (!(this.source instanceof BinaryUnionPlanNode)) {
+ throw new IllegalStateException();
+ } else {
+ this.source = newUnionNode;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public int getMaxDepth() {
+ return this.source.getOptimizerNode().getMaxDepth() + 1;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ return "Channel (" + this.source + (this.target == null ? ')' : ") -> (" + this.target + ')') +
+ '[' + this.shipStrategy + "] [" + this.localStrategy + "] " +
+ (this.tempMode == null || this.tempMode == TempMode.NONE ? "{NO-TEMP}" : this.tempMode);
+ }
+
+ @Override
+ public Channel clone() {
+ try {
+ return (Channel) super.clone();
+ } catch (CloneNotSupportedException cnsex) {
+ throw new RuntimeException(cnsex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
new file mode 100644
index 0000000..01c56dd
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/DualInputPlanNode.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+
+/**
+ *
+ */
+public class DualInputPlanNode extends PlanNode {
+
+ protected final Channel input1;
+ protected final Channel input2;
+
+ protected final FieldList keys1;
+ protected final FieldList keys2;
+
+ protected final boolean[] sortOrders;
+
+ private TypeComparatorFactory<?> comparator1;
+ private TypeComparatorFactory<?> comparator2;
+ private TypePairComparatorFactory<?, ?> pairComparator;
+
+ public Object postPassHelper1;
+ public Object postPassHelper2;
+
+ // --------------------------------------------------------------------------------------------
+
+ public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy) {
+ this(template, nodeName, input1, input2, diverStrategy, null, null, null);
+ }
+
+ public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2,
+ DriverStrategy diverStrategy, FieldList driverKeyFields1, FieldList driverKeyFields2)
+ {
+ this(template, nodeName, input1, input2, diverStrategy, driverKeyFields1, driverKeyFields2,
+ SingleInputPlanNode.getTrueArray(driverKeyFields1.size()));
+ }
+
+ public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy,
+ FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders)
+ {
+ super(template, nodeName, diverStrategy);
+ this.input1 = input1;
+ this.input2 = input2;
+ this.keys1 = driverKeyFields1;
+ this.keys2 = driverKeyFields2;
+ this.sortOrders = driverSortOrders;
+
+ if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
+ this.input1.setReplicationFactor(getParallelism());
+ }
+ if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
+ this.input2.setReplicationFactor(getParallelism());
+ }
+
+ mergeBranchPlanMaps(input1.getSource(), input2.getSource());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public TwoInputNode getTwoInputNode() {
+ if (this.template instanceof TwoInputNode) {
+ return (TwoInputNode) this.template;
+ } else {
+ throw new RuntimeException();
+ }
+ }
+
+ public FieldList getKeysForInput1() {
+ return this.keys1;
+ }
+
+ public FieldList getKeysForInput2() {
+ return this.keys2;
+ }
+
+ public boolean[] getSortOrders() {
+ return this.sortOrders;
+ }
+
+ public TypeComparatorFactory<?> getComparator1() {
+ return this.comparator1;
+ }
+
+ public TypeComparatorFactory<?> getComparator2() {
+ return this.comparator2;
+ }
+
+ public void setComparator1(TypeComparatorFactory<?> comparator) {
+ this.comparator1 = comparator;
+ }
+
+ public void setComparator2(TypeComparatorFactory<?> comparator) {
+ this.comparator2 = comparator;
+ }
+
+ public TypePairComparatorFactory<?, ?> getPairComparator() {
+ return this.pairComparator;
+ }
+
+ public void setPairComparator(TypePairComparatorFactory<?, ?> comparator) {
+ this.pairComparator = comparator;
+ }
+
+ /**
+ * Gets the first input channel to this node.
+ *
+ * @return The first input channel to this node.
+ */
+ public Channel getInput1() {
+ return this.input1;
+ }
+
+ /**
+ * Gets the second input channel to this node.
+ *
+ * @return The second input channel to this node.
+ */
+ public Channel getInput2() {
+ return this.input2;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ this.input1.getSource().accept(visitor);
+ this.input2.getSource().accept(visitor);
+
+ for (Channel broadcastInput : getBroadcastInputs()) {
+ broadcastInput.getSource().accept(visitor);
+ }
+
+ visitor.postVisit(this);
+ }
+ }
+
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) {
+ return Arrays.asList(this.input1.getSource(), this.input2.getSource());
+ } else {
+ List<PlanNode> preds = new ArrayList<PlanNode>();
+
+ preds.add(input1.getSource());
+ preds.add(input2.getSource());
+
+ for (Channel c : getBroadcastInputs()) {
+ preds.add(c.getSource());
+ }
+
+ return preds;
+ }
+ }
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Arrays.asList(this.input1, this.input2);
+ }
+
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+
+ // check first input
+ SourceAndDamReport res1 = this.input1.getSource().hasDamOnPathDownTo(source);
+ if (res1 == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (res1 == FOUND_SOURCE) {
+ if (this.input1.getLocalStrategy().dams() || this.input1.getTempMode().breaksPipeline() ||
+ getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ return FOUND_SOURCE;
+ }
+ }
+ else {
+ SourceAndDamReport res2 = this.input2.getSource().hasDamOnPathDownTo(source);
+ if (res2 == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (res2 == FOUND_SOURCE) {
+ if (this.input2.getLocalStrategy().dams() || this.input2.getTempMode().breaksPipeline() ||
+ getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ return FOUND_SOURCE;
+ }
+ }
+ else {
+ // NOT_FOUND
+ // check the broadcast inputs
+
+ for (NamedChannel nc : getBroadcastInputs()) {
+ SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source);
+ if (bcRes != NOT_FOUND) {
+ // broadcast inputs are always dams
+ return FOUND_SOURCE_AND_DAM;
+ }
+ }
+ return NOT_FOUND;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
new file mode 100644
index 0000000..d146c83
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/FlinkPlan.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+/**
+ * A common interface for compiled Flink plans for both batch and streaming
+ * processing programs.
+ *
+ */
+public interface FlinkPlan {
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
new file mode 100644
index 0000000..38f76b2
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/IterationPlanNode.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ *
+ */
+public interface IterationPlanNode {
+
+ void acceptForStepFunction(Visitor<PlanNode> visitor);
+
+ IterationNode getIterationNode();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
new file mode 100644
index 0000000..3650eea
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NAryUnionPlanNode.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.IterableIterator;
+import org.apache.flink.util.Visitor;
+
+/**
+ * A union operation over multiple inputs (2 or more).
+ */
+public class NAryUnionPlanNode extends PlanNode {
+
+ private final List<Channel> inputs;
+
+ /**
+ * @param template
+ */
+ public NAryUnionPlanNode(BinaryUnionNode template, List<Channel> inputs, GlobalProperties gProps,
+ Costs cumulativeCosts)
+ {
+ super(template, "Union", DriverStrategy.NONE);
+
+ this.inputs = inputs;
+ this.globalProps = gProps;
+ this.localProps = new LocalProperties();
+ this.nodeCosts = new Costs();
+ this.cumulativeCosts = cumulativeCosts;
+ }
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ visitor.preVisit(this);
+ for (Channel c : this.inputs) {
+ c.getSource().accept(visitor);
+ }
+ visitor.postVisit(this);
+ }
+
+ public List<Channel> getListOfInputs() {
+ return this.inputs;
+ }
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.unmodifiableList(this.inputs);
+ }
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ final Iterator<Channel> channels = this.inputs.iterator();
+ return new IterableIterator<PlanNode>() {
+
+ @Override
+ public boolean hasNext() {
+ return channels.hasNext();
+ }
+
+ @Override
+ public PlanNode next() {
+ return channels.next().getSource();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<PlanNode> iterator() {
+ return this;
+ }
+ };
+ }
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ // this node is used after the plan enumeration. consequently, this will never be invoked here
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
new file mode 100644
index 0000000..da97e61
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/NamedChannel.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.optimizer.dag.TempMode;
+
+public class NamedChannel extends Channel {
+
+ private final String name;
+
+ /**
+ * Initializes NamedChannel.
+ *
+ * @param sourceNode
+ */
+ public NamedChannel(String name, PlanNode sourceNode) {
+ super(sourceNode);
+ this.name = name;
+ }
+
+ public NamedChannel(String name, PlanNode sourceNode, TempMode tempMode) {
+ super(sourceNode, tempMode);
+ this.name = name;
+ }
+
+ public String getName() {
+ return this.name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
new file mode 100644
index 0000000..d56be87
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import java.util.Collection;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.util.Visitable;
+import org.apache.flink.util.Visitor;
+
+/**
+ * The execution plan generated by the Optimizer. It contains {@link PlanNode}s
+ * and {@link Channel}s that describe exactly how the program should be executed.
+ * It defines all ship strategies (local pipe, shuffle, broadcast, rebalance), all
+ * operator strategies (sorting-merge join, hash join, sorted grouping, ...),
+ * and the data exchange modes (batched, pipelined).
+ */
+public class OptimizedPlan implements FlinkPlan, Visitable<PlanNode> {
+
+ /** The data sources in the plan. */
+ private final Collection<SourcePlanNode> dataSources;
+
+ /** The data sinks in the plan. */
+ private final Collection<SinkPlanNode> dataSinks;
+
+ /** All nodes in the optimizer plan. */
+ private final Collection<PlanNode> allNodes;
+
+ /** The original program. */
+ private final Plan originalProgram;
+
+ /** Name of the job */
+ private final String jobName;
+
+ /**
+ * Creates a new instance of this optimizer plan container. The plan is given and fully
+ * described by the data sources, sinks and the collection of all nodes.
+ *
+ * @param sources The data sources.
+ * @param sinks The data sinks.
+ * @param allNodes A collection containing all nodes in the plan.
+ * @param jobName The name of the program
+ */
+ public OptimizedPlan(Collection<SourcePlanNode> sources, Collection<SinkPlanNode> sinks,
+ Collection<PlanNode> allNodes, String jobName, Plan programPlan)
+ {
+ this.dataSources = sources;
+ this.dataSinks = sinks;
+ this.allNodes = allNodes;
+ this.jobName = jobName;
+ this.originalProgram = programPlan;
+ }
+
+ /**
+ * Gets the data sources from this OptimizedPlan.
+ *
+ * @return The data sources.
+ */
+ public Collection<SourcePlanNode> getDataSources() {
+ return dataSources;
+ }
+
+ /**
+ * Gets the data sinks from this OptimizedPlan.
+ *
+ * @return The data sinks.
+ */
+ public Collection<SinkPlanNode> getDataSinks() {
+ return dataSinks;
+ }
+
+ /**
+ * Gets all the nodes from this OptimizedPlan.
+ *
+ * @return All nodes.
+ */
+ public Collection<PlanNode> getAllNodes() {
+ return allNodes;
+ }
+
+ /**
+ * Returns the name of the program.
+ *
+ * @return The name of the program.
+ */
+ public String getJobName() {
+ return this.jobName;
+ }
+
+ /**
+ * Gets the original program plan from which this optimized plan was created.
+ *
+ * @return The original program plan.
+ */
+ public Plan getOriginalPactPlan() {
+ return this.originalProgram;
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Applies the given visitor top down to all nodes, starting at the sinks.
+ *
+ * @param visitor
+ * The visitor to apply to the nodes in this plan.
+ * @see org.apache.flink.util.Visitable#accept(org.apache.flink.util.Visitor)
+ */
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ for (SinkPlanNode node : this.dataSinks) {
+ node.accept(visitor);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
new file mode 100644
index 0000000..6f634fb
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.OptimizerNode.UnclosedBranchDescriptor;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plandump.DumpableConnection;
+import org.apache.flink.optimizer.plandump.DumpableNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.util.Visitable;
+
+/**
+ * The representation of a data exchange between to operators. The data exchange can realize a shipping strategy,
+ * which established global properties, and a local strategy, which establishes local properties.
+ * <p>
+ * Because we currently deal only with plans where the operator order is fixed, many properties are equal
+ * among candidates and are determined prior to the enumeration (such as for example constant/dynamic path membership).
+ * Hence, many methods will delegate to the {@code OptimizerNode} that represents the node this candidate was
+ * created for.
+ */
+public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<PlanNode> {
+
+ protected final OptimizerNode template;
+
+ protected final List<Channel> outChannels;
+
+ private List<NamedChannel> broadcastInputs;
+
+ private final String nodeName;
+
+ private DriverStrategy driverStrategy; // The local strategy (sorting / hashing, ...)
+
+ protected LocalProperties localProps; // local properties of the data produced by this node
+
+ protected GlobalProperties globalProps; // global properties of the data produced by this node
+
+ protected Map<OptimizerNode, PlanNode> branchPlan; // the actual plan alternative chosen at a branch point
+
+ protected Costs nodeCosts; // the costs incurred by this node
+
+ protected Costs cumulativeCosts; // the cumulative costs of all operators in the sub-tree
+
+ private double relativeMemoryPerSubTask; // the amount of memory dedicated to each task, in bytes
+
+ private int parallelism;
+
+ private boolean pFlag; // flag for the internal pruning algorithm
+
+ // --------------------------------------------------------------------------------------------
+
+ public PlanNode(OptimizerNode template, String nodeName, DriverStrategy strategy) {
+ this.outChannels = new ArrayList<Channel>(2);
+ this.broadcastInputs = new ArrayList<NamedChannel>();
+ this.template = template;
+ this.nodeName = nodeName;
+ this.driverStrategy = strategy;
+
+ this.parallelism = template.getParallelism();
+
+ // check, if there is branch at this node. if yes, this candidate must be associated with
+ // the branching template node.
+ if (template.isBranching()) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
+ this.branchPlan.put(template, this);
+ }
+ }
+
+ protected void mergeBranchPlanMaps(PlanNode pred1, PlanNode pred2) {
+ mergeBranchPlanMaps(pred1.branchPlan, pred2.branchPlan);
+ }
+
+ protected void mergeBranchPlanMaps(Map<OptimizerNode, PlanNode> branchPlan1, Map<OptimizerNode, PlanNode> branchPlan2) {
+ // merge the branchPlan maps according the template's uncloseBranchesStack
+ if (this.template.hasUnclosedBranches()) {
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
+ }
+
+ for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
+ OptimizerNode brancher = uc.getBranchingNode();
+ PlanNode selectedCandidate = null;
+
+ if (branchPlan1 != null) {
+ // predecessor 1 has branching children, see if it got the branch we are looking for
+ selectedCandidate = branchPlan1.get(brancher);
+ }
+
+ if (selectedCandidate == null && branchPlan2 != null) {
+ // predecessor 2 has branching children, see if it got the branch we are looking for
+ selectedCandidate = branchPlan2.get(brancher);
+ }
+
+ // it may be that the branch candidate is only found once the broadcast variables are set
+ if (selectedCandidate != null) {
+ this.branchPlan.put(brancher, selectedCandidate);
+ }
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Accessors
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the node from the optimizer DAG for which this plan candidate node was created.
+ *
+ * @return The optimizer's DAG node.
+ */
+ public OptimizerNode getOriginalOptimizerNode() {
+ return this.template;
+ }
+
+ /**
+ * Gets the program operator that this node represents in the plan.
+ *
+ * @return The program operator this node represents in the plan.
+ */
+ public Operator<?> getProgramOperator() {
+ return this.template.getOperator();
+ }
+
+ /**
+ * Gets the name of the plan node.
+ *
+ * @return The name of the plan node.
+ */
+ public String getNodeName() {
+ return this.nodeName;
+ }
+
+ public int getMemoryConsumerWeight() {
+ return this.driverStrategy.isMaterializing() ? 1 : 0;
+ }
+
+ /**
+ * Gets the memory dedicated to each sub-task for this node.
+ *
+ * @return The memory per task, in bytes.
+ */
+ public double getRelativeMemoryPerSubTask() {
+ return this.relativeMemoryPerSubTask;
+ }
+
+ /**
+ * Sets the memory dedicated to each task for this node.
+ *
+ * @param relativeMemoryPerSubtask The relative memory per sub-task
+ */
+ public void setRelativeMemoryPerSubtask(double relativeMemoryPerSubtask) {
+ this.relativeMemoryPerSubTask = relativeMemoryPerSubtask;
+ }
+
+ /**
+ * Gets the driver strategy from this node. This determines for example for a <i>match</i> Pact whether
+ * to use a merge or a hybrid hash strategy.
+ *
+ * @return The driver strategy.
+ */
+ public DriverStrategy getDriverStrategy() {
+ return this.driverStrategy;
+ }
+
+ /**
+ * Sets the driver strategy for this node. Usually should not be changed.
+ *
+ * @param newDriverStrategy The driver strategy.
+ */
+ public void setDriverStrategy(DriverStrategy newDriverStrategy) {
+ this.driverStrategy = newDriverStrategy;
+ }
+
+ public void initProperties(GlobalProperties globals, LocalProperties locals) {
+ if (this.globalProps != null || this.localProps != null) {
+ throw new IllegalStateException();
+ }
+ this.globalProps = globals;
+ this.localProps = locals;
+ }
+
+ /**
+ * Gets the local properties from this PlanNode.
+ *
+ * @return The local properties.
+ */
+ public LocalProperties getLocalProperties() {
+ return this.localProps;
+ }
+
+ /**
+ * Gets the global properties from this PlanNode.
+ *
+ * @return The global properties.
+ */
+ public GlobalProperties getGlobalProperties() {
+ return this.globalProps;
+ }
+
+ /**
+ * Gets the costs incurred by this node. The costs reflect also the costs incurred by the shipping strategies
+ * of the incoming connections.
+ *
+ * @return The node-costs, or null, if not yet set.
+ */
+ public Costs getNodeCosts() {
+ return this.nodeCosts;
+ }
+
+ /**
+ * Gets the cumulative costs of this nose. The cumulative costs are the sum of the costs
+ * of this node and of all nodes in the subtree below this node.
+ *
+ * @return The cumulative costs, or null, if not yet set.
+ */
+ public Costs getCumulativeCosts() {
+ return this.cumulativeCosts;
+ }
+
+ public Costs getCumulativeCostsShare() {
+ if (this.cumulativeCosts == null) {
+ return null;
+ } else {
+ Costs result = cumulativeCosts.clone();
+ if (this.template.getOutgoingConnections() != null) {
+ int outDegree = this.template.getOutgoingConnections().size();
+ if (outDegree > 0) {
+ result.divideBy(outDegree);
+ }
+ }
+
+ return result;
+ }
+ }
+
+
+ /**
+ * Sets the basic cost for this node to the given value, and sets the cumulative costs
+ * to those costs plus the cost shares of all inputs (regular and broadcast).
+ *
+ * @param nodeCosts The already knows costs for this node
+ * (this cost a produces by a concrete {@code OptimizerNode} subclass.
+ */
+ public void setCosts(Costs nodeCosts) {
+ // set the node costs
+ this.nodeCosts = nodeCosts;
+
+ // the cumulative costs are the node costs plus the costs of all inputs
+ this.cumulativeCosts = nodeCosts.clone();
+
+ // add all the normal inputs
+ for (PlanNode pred : getPredecessors()) {
+
+ Costs parentCosts = pred.getCumulativeCostsShare();
+ if (parentCosts != null) {
+ this.cumulativeCosts.addCosts(parentCosts);
+ } else {
+ throw new CompilerException("Trying to set the costs of an operator before the predecessor costs are computed.");
+ }
+ }
+
+ // add all broadcast variable inputs
+ if (this.broadcastInputs != null) {
+ for (NamedChannel nc : this.broadcastInputs) {
+ Costs bcInputCost = nc.getSource().getCumulativeCostsShare();
+ if (bcInputCost != null) {
+ this.cumulativeCosts.addCosts(bcInputCost);
+ } else {
+ throw new CompilerException("Trying to set the costs of an operator before the broadcast input costs are computed.");
+ }
+ }
+ }
+ }
+
+ public void setParallelism(int parallelism) {
+ this.parallelism = parallelism;
+ }
+
+ public int getParallelism() {
+ return this.parallelism;
+ }
+
+ public long getGuaranteedAvailableMemory() {
+ return this.template.getMinimalMemoryAcrossAllSubTasks();
+ }
+
+ public Map<OptimizerNode, PlanNode> getBranchPlan() {
+ return branchPlan;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Input, Predecessors, Successors
+ // --------------------------------------------------------------------------------------------
+
+ public abstract Iterable<Channel> getInputs();
+
+ @Override
+ public abstract Iterable<PlanNode> getPredecessors();
+
+ /**
+ * Sets a list of all broadcast inputs attached to this node.
+ */
+ public void setBroadcastInputs(List<NamedChannel> broadcastInputs) {
+ if (broadcastInputs != null) {
+ this.broadcastInputs = broadcastInputs;
+
+ // update the branch map
+ for (NamedChannel nc : broadcastInputs) {
+ PlanNode source = nc.getSource();
+
+ mergeBranchPlanMaps(branchPlan, source.branchPlan);
+ }
+ }
+
+ // do a sanity check that if we are branching, we have now candidates for each branch point
+ if (this.template.hasUnclosedBranches()) {
+ if (this.branchPlan == null) {
+ throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
+ }
+
+ for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
+ OptimizerNode brancher = uc.getBranchingNode();
+ if (this.branchPlan.get(brancher) == null) {
+ throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets a list of all broadcast inputs attached to this node.
+ */
+ public List<NamedChannel> getBroadcastInputs() {
+ return this.broadcastInputs;
+ }
+
+ /**
+ * Adds a channel to a successor node to this node.
+ *
+ * @param channel The channel to the successor.
+ */
+ public void addOutgoingChannel(Channel channel) {
+ this.outChannels.add(channel);
+ }
+
+ /**
+ * Gets a list of all outgoing channels leading to successors.
+ *
+ * @return A list of all channels leading to successors.
+ */
+ public List<Channel> getOutgoingChannels() {
+ return this.outChannels;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Miscellaneous
+ // --------------------------------------------------------------------------------------------
+
+ public void updatePropertiesWithUniqueSets(Set<FieldSet> uniqueFieldCombinations) {
+ if (uniqueFieldCombinations == null || uniqueFieldCombinations.isEmpty()) {
+ return;
+ }
+ for (FieldSet fields : uniqueFieldCombinations) {
+ this.globalProps.addUniqueFieldCombination(fields);
+ this.localProps = this.localProps.addUniqueFields(fields);
+ }
+ }
+
+ public PlanNode getCandidateAtBranchPoint(OptimizerNode branchPoint) {
+ if (branchPlan == null) {
+ return null;
+ } else {
+ return this.branchPlan.get(branchPoint);
+ }
+ }
+
+ /**
+ * Sets the pruning marker to true.
+ */
+ public void setPruningMarker() {
+ this.pFlag = true;
+ }
+
+ /**
+ * Checks whether the pruning marker was set.
+ *
+ * @return True, if the pruning marker was set, false otherwise.
+ */
+ public boolean isPruneMarkerSet() {
+ return this.pFlag;
+ }
+
+ public boolean isOnDynamicPath() {
+ return this.template.isOnDynamicPath();
+ }
+
+ public int getCostWeight() {
+ return this.template.getCostWeight();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Checks whether this node has a dam on the way down to the given source node. This method
+ * returns either that (a) the source node is not found as a (transitive) child of this node,
+ * (b) the node is found, but no dam is on the path, or (c) the node is found and a dam is on
+ * the path.
+ *
+ * @param source The node on the path to which the dam is sought.
+ * @return The result whether the node is found and whether a dam is on the path.
+ */
+ public abstract SourceAndDamReport hasDamOnPathDownTo(PlanNode source);
+
+ public FeedbackPropertiesMeetRequirementsReport checkPartialSolutionPropertiesMet(PlanNode partialSolution, GlobalProperties feedbackGlobal, LocalProperties feedbackLocal) {
+ if (this == partialSolution) {
+ return FeedbackPropertiesMeetRequirementsReport.PENDING;
+ }
+
+ boolean found = false;
+ boolean allMet = true;
+ boolean allLocallyMet = true;
+
+ for (Channel input : getInputs()) {
+ FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
+
+ if (inputState == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
+ continue;
+ }
+ else if (inputState == FeedbackPropertiesMeetRequirementsReport.MET) {
+ found = true;
+ continue;
+ }
+ else if (inputState == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
+ return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
+ }
+ else {
+ found = true;
+
+ // the partial solution was on the path here. check whether the channel requires
+ // certain properties that are met, or whether the channel introduces new properties
+
+ // if the plan introduces new global properties, then we can stop looking whether
+ // the feedback properties are sufficient to meet the requirements
+ if (input.getShipStrategy() != ShipStrategyType.FORWARD && input.getShipStrategy() != ShipStrategyType.NONE) {
+ continue;
+ }
+
+ // first check whether this channel requires something that is not met
+ if (input.getRequiredGlobalProps() != null && !input.getRequiredGlobalProps().isMetBy(feedbackGlobal)) {
+ return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
+ }
+
+ // in general, not everything is met here already
+ allMet = false;
+
+ // if the plan introduces new local properties, we can stop checking for matching local properties
+ if (inputState != FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET) {
+
+ if (input.getLocalStrategy() == LocalStrategy.NONE) {
+
+ if (input.getRequiredLocalProps() != null && !input.getRequiredLocalProps().isMetBy(feedbackLocal)) {
+ return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
+ }
+
+ allLocallyMet = false;
+ }
+ }
+ }
+ }
+
+ if (!found) {
+ return FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION;
+ } else if (allMet) {
+ return FeedbackPropertiesMeetRequirementsReport.MET;
+ } else if (allLocallyMet) {
+ return FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET;
+ } else {
+ return FeedbackPropertiesMeetRequirementsReport.PENDING;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public String toString() {
+ return this.template.getName() + " \"" + getProgramOperator().getName() + "\" : " + this.driverStrategy +
+ " [[ " + this.globalProps + " ]] [[ " + this.localProps + " ]]";
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public OptimizerNode getOptimizerNode() {
+ return this.template;
+ }
+
+ @Override
+ public PlanNode getPlanNode() {
+ return this;
+ }
+
+ @Override
+ public Iterable<DumpableConnection<PlanNode>> getDumpableInputs() {
+ List<DumpableConnection<PlanNode>> allInputs = new ArrayList<DumpableConnection<PlanNode>>();
+
+ for (Channel c : getInputs()) {
+ allInputs.add(c);
+ }
+
+ for (NamedChannel c : getBroadcastInputs()) {
+ allInputs.add(c);
+ }
+
+ return allInputs;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static enum SourceAndDamReport {
+ NOT_FOUND, FOUND_SOURCE, FOUND_SOURCE_AND_DAM;
+ }
+
+
+
+ public static enum FeedbackPropertiesMeetRequirementsReport {
+ /** Indicates that the path is irrelevant */
+ NO_PARTIAL_SOLUTION,
+
+ /** Indicates that the question whether the properties are met has been determined pending
+ * dependent on global and local properties */
+ PENDING,
+
+ /** Indicates that the question whether the properties are met has been determined pending
+ * dependent on global properties only */
+ PENDING_LOCAL_MET,
+
+ /** Indicates that the question whether the properties are met has been determined true */
+ MET,
+
+ /** Indicates that the question whether the properties are met has been determined false */
+ NOT_MET;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
new file mode 100644
index 0000000..b928be7
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.runtime.operators.DamBehavior;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Visitor;
+
+/**
+ *
+ */
+public class SingleInputPlanNode extends PlanNode {
+
+ protected final Channel input;
+
+ protected final FieldList[] driverKeys;
+
+ protected final boolean[][] driverSortOrders;
+
+ private TypeComparatorFactory<?>[] comparators;
+
+ public Object postPassHelper;
+
+ // --------------------------------------------------------------------------------------------
+
+ public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy) {
+ this(template, nodeName, input, driverStrategy, null, null);
+ }
+
+ public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input,
+ DriverStrategy driverStrategy, FieldList driverKeyFields)
+ {
+ this(template, nodeName, input, driverStrategy, driverKeyFields, getTrueArray(driverKeyFields.size()));
+ }
+
+ public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input,
+ DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
+ {
+ super(template, nodeName, driverStrategy);
+ this.input = input;
+
+ this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
+ this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
+ this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
+
+ if(driverStrategy.getNumRequiredComparators() > 0) {
+ this.driverKeys[0] = driverKeyFields;
+ this.driverSortOrders[0] = driverSortOrders;
+ }
+
+ if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
+ this.input.setReplicationFactor(getParallelism());
+ }
+
+ final PlanNode predNode = input.getSource();
+
+ if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
+
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
+ }
+ this.branchPlan.putAll(predNode.branchPlan);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public SingleInputNode getSingleInputNode() {
+ if (this.template instanceof SingleInputNode) {
+ return (SingleInputNode) this.template;
+ } else {
+ throw new RuntimeException();
+ }
+ }
+
+ /**
+ * Gets the input channel to this node.
+ *
+ * @return The input channel to this node.
+ */
+ public Channel getInput() {
+ return this.input;
+ }
+
+ /**
+ * Gets the predecessor of this node, i.e. the source of the input channel.
+ *
+ * @return The predecessor of this node.
+ */
+ public PlanNode getPredecessor() {
+ return this.input.getSource();
+ }
+
+ /**
+ * Sets the key field indexes for the specified driver comparator.
+ *
+ * @param keys The key field indexes for the specified driver comparator.
+ * @param id The ID of the driver comparator.
+ */
+ public void setDriverKeyInfo(FieldList keys, int id) {
+ this.setDriverKeyInfo(keys, getTrueArray(keys.size()), id);
+ }
+
+ /**
+ * Sets the key field information for the specified driver comparator.
+ *
+ * @param keys The key field indexes for the specified driver comparator.
+ * @param sortOrder The key sort order for the specified driver comparator.
+ * @param id The ID of the driver comparator.
+ */
+ public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) {
+ if(id < 0 || id >= driverKeys.length) {
+ throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only "
+ +super.getDriverStrategy().getNumRequiredComparators()+" comparators.");
+ }
+ this.driverKeys[id] = keys;
+ this.driverSortOrders[id] = sortOrder;
+ }
+
+ /**
+ * Gets the key field indexes for the specified driver comparator.
+ *
+ * @param id The id of the driver comparator for which the key field indexes are requested.
+ * @return The key field indexes of the specified driver comparator.
+ */
+ public FieldList getKeys(int id) {
+ return this.driverKeys[id];
+ }
+
+ /**
+ * Gets the sort order for the specified driver comparator.
+ *
+ * @param id The id of the driver comparator for which the sort order is requested.
+ * @return The sort order of the specified driver comparator.
+ */
+ public boolean[] getSortOrders(int id) {
+ return driverSortOrders[id];
+ }
+
+ /**
+ * Gets the specified comparator from this PlanNode.
+ *
+ * @param id The ID of the requested comparator.
+ *
+ * @return The specified comparator.
+ */
+ public TypeComparatorFactory<?> getComparator(int id) {
+ return comparators[id];
+ }
+
+ /**
+ * Sets the specified comparator for this PlanNode.
+ *
+ * @param comparator The comparator to set.
+ * @param id The ID of the comparator to set.
+ */
+ public void setComparator(TypeComparatorFactory<?> comparator, int id) {
+ this.comparators[id] = comparator;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ this.input.getSource().accept(visitor);
+
+ for (Channel broadcastInput : getBroadcastInputs()) {
+ broadcastInput.getSource().accept(visitor);
+ }
+
+ visitor.postVisit(this);
+ }
+ }
+
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) {
+ return Collections.singleton(this.input.getSource());
+ }
+ else {
+ List<PlanNode> preds = new ArrayList<PlanNode>();
+ preds.add(input.getSource());
+
+ for (Channel c : getBroadcastInputs()) {
+ preds.add(c.getSource());
+ }
+
+ return preds;
+ }
+ }
+
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.singleton(this.input);
+ }
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+ SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source);
+ if (res == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (res == FOUND_SOURCE) {
+ return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() ||
+ getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
+ FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
+ }
+ else {
+ // NOT_FOUND
+ // check the broadcast inputs
+
+ for (NamedChannel nc : getBroadcastInputs()) {
+ SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source);
+ if (bcRes != NOT_FOUND) {
+ // broadcast inputs are always dams
+ return FOUND_SOURCE_AND_DAM;
+ }
+ }
+ return NOT_FOUND;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ protected static boolean[] getTrueArray(int length) {
+ final boolean[] a = new boolean[length];
+ for (int i = 0; i < length; i++) {
+ a[i] = true;
+ }
+ return a;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
new file mode 100644
index 0000000..451484d
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import java.util.List;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.SinkJoiner;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+/**
+ *
+ */
+public class SinkJoinerPlanNode extends DualInputPlanNode {
+
+ public SinkJoinerPlanNode(SinkJoiner template, Channel input1, Channel input2) {
+ super(template, "", input1, input2, DriverStrategy.BINARY_NO_OP);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setCosts(Costs nodeCosts) {
+ // the plan enumeration logic works as for regular two-input-operators, which is important
+ // because of the branch handling logic. it does pick redistributing network channels
+ // between the sink and the sink joiner, because sinks joiner has a different DOP than the sink.
+ // we discard any cost and simply use the sum of the costs from the two children.
+
+ Costs totalCosts = getInput1().getSource().getCumulativeCosts().clone();
+ totalCosts.addCosts(getInput2().getSource().getCumulativeCosts());
+ super.setCosts(totalCosts);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void getDataSinks(List<SinkPlanNode> sinks) {
+ final PlanNode in1 = this.input1.getSource();
+ final PlanNode in2 = this.input2.getSource();
+
+ if (in1 instanceof SinkPlanNode) {
+ sinks.add((SinkPlanNode) in1);
+ } else if (in1 instanceof SinkJoinerPlanNode) {
+ ((SinkJoinerPlanNode) in1).getDataSinks(sinks);
+ } else {
+ throw new CompilerException("Illegal child node for a sink joiner utility node: Neither Sink nor Sink Joiner");
+ }
+
+ if (in2 instanceof SinkPlanNode) {
+ sinks.add((SinkPlanNode) in2);
+ } else if (in2 instanceof SinkJoinerPlanNode) {
+ ((SinkJoinerPlanNode) in2).getDataSinks(sinks);
+ } else {
+ throw new CompilerException("Illegal child node for a sink joiner utility node: Neither Sink nor Sink Joiner");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
new file mode 100644
index 0000000..656e67f
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.optimizer.dag.DataSinkNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ * Plan candidate node for data flow sinks.
+ */
+public class SinkPlanNode extends SingleInputPlanNode
+{
+ /**
+ * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that
+ * local sorting and range partitioning are handled by the incoming channel already.
+ *
+ * @param template The template optimizer node that this candidate is created for.
+ */
+ public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) {
+ super(template, nodeName, input, DriverStrategy.NONE);
+
+ this.globalProps = input.getGlobalProperties().clone();
+ this.localProps = input.getLocalProperties().clone();
+ }
+
+ public DataSinkNode getSinkNode() {
+ if (this.template instanceof DataSinkNode) {
+ return (DataSinkNode) this.template;
+ } else {
+ throw new RuntimeException();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
new file mode 100644
index 0000000..63093dd
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SolutionSetNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Plan candidate node for partial solution of a bulk iteration.
+ */
+public class SolutionSetPlanNode extends PlanNode {
+
+ private static final Costs NO_COSTS = new Costs();
+
+ private WorksetIterationPlanNode containingIterationNode;
+
+ private final Channel initialInput;
+
+ public Object postPassHelper;
+
+
+ public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
+ GlobalProperties gProps, LocalProperties lProps,
+ Channel initialInput)
+ {
+ super(template, nodeName, DriverStrategy.NONE);
+
+ this.globalProps = gProps;
+ this.localProps = lProps;
+ this.initialInput = initialInput;
+
+ // the node incurs no cost
+ this.nodeCosts = NO_COSTS;
+ this.cumulativeCosts = NO_COSTS;
+
+ if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
+ if (this.branchPlan == null) {
+ this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
+ }
+
+ this.branchPlan.putAll(initialInput.getSource().branchPlan);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public SolutionSetNode getSolutionSetNode() {
+ return (SolutionSetNode) this.template;
+ }
+
+ public WorksetIterationPlanNode getContainingIterationNode() {
+ return this.containingIterationNode;
+ }
+
+ public void setContainingIterationNode(WorksetIterationPlanNode containingIterationNode) {
+ this.containingIterationNode = containingIterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ return Collections.<PlanNode>emptyList();
+ }
+
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.<Channel>emptyList();
+ }
+
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+
+ SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
+ if (res == FOUND_SOURCE_AND_DAM || res == FOUND_SOURCE) {
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ return NOT_FOUND;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
new file mode 100644
index 0000000..11b7cc9
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
+
+import java.util.Collections;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.dag.DataSourceNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Plan candidate node for data flow sources that have no input and no special strategies.
+ */
+public class SourcePlanNode extends PlanNode {
+
+ private TypeSerializerFactory<?> serializer;
+
+ /**
+ * Constructs a new source candidate node that uses <i>NONE</i> as its local strategy.
+ *
+ * @param template The template optimizer node that this candidate is created for.
+ */
+ public SourcePlanNode(DataSourceNode template, String nodeName) {
+ this(template, nodeName, new GlobalProperties(), new LocalProperties());
+ }
+
+ public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) {
+ super(template, nodeName, DriverStrategy.NONE);
+
+ this.globalProps = gprops;
+ this.localProps = lprops;
+ updatePropertiesWithUniqueSets(template.getUniqueFields());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public DataSourceNode getDataSourceNode() {
+ return (DataSourceNode) this.template;
+ }
+
+ /**
+ * Gets the serializer from this PlanNode.
+ *
+ * @return The serializer.
+ */
+ public TypeSerializerFactory<?> getSerializer() {
+ return serializer;
+ }
+
+ /**
+ * Sets the serializer for this PlanNode.
+ *
+ * @param serializer The serializer to set.
+ */
+ public void setSerializer(TypeSerializerFactory<?> serializer) {
+ this.serializer = serializer;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public void accept(Visitor<PlanNode> visitor) {
+ if (visitor.preVisit(this)) {
+ visitor.postVisit(this);
+ }
+ }
+
+
+ @Override
+ public Iterable<PlanNode> getPredecessors() {
+ return Collections.<PlanNode>emptyList();
+ }
+
+
+ @Override
+ public Iterable<Channel> getInputs() {
+ return Collections.<Channel>emptyList();
+ }
+
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ } else {
+ return NOT_FOUND;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
new file mode 100644
index 0000000..880f2e3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.flink.runtime.jobgraph.JobGraph;
+
+/**
+ * Abstract class representing Flink Streaming plans
+ *
+ */
+public abstract class StreamingPlan implements FlinkPlan {
+
+ public abstract JobGraph getJobGraph();
+
+ public abstract String getStreamingPlanAsJSON();
+
+ public abstract void dumpStreamingPlanAsJSON(File file) throws IOException;
+
+}
[43/53] [abbrv] flink git commit: [FLINK-441] [optimizer] Remove
obsolete and unused utility classes
Posted by se...@apache.org.
[FLINK-441] [optimizer] Remove obsolete and unused utility classes
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9d222ca1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9d222ca1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9d222ca1
Branch: refs/heads/master
Commit: 9d222ca1bb77b48e0988d5d1a9c3cb7f8fc622be
Parents: c91fb7c
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 11:02:33 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
.../optimizer/NonCachingDataStatistics.java | 35 -----
.../optimizer/postpass/DenseValueSchema.java | 156 -------------------
.../postpass/MissingFieldTypeInfoException.java | 1 -
.../optimizer/postpass/SparseKeySchema.java | 1 -
4 files changed, 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9d222ca1/flink-compiler/src/main/java/org/apache/flink/optimizer/NonCachingDataStatistics.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/NonCachingDataStatistics.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/NonCachingDataStatistics.java
deleted file mode 100644
index 3e1a1dd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/NonCachingDataStatistics.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer;
-
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-
-/**
- * A variant of the data statistics that never caches.
- */
-public class NonCachingDataStatistics extends DataStatistics {
-
- public BaseStatistics getBaseStatistics(String inputIdentifier) {
- return null;
- }
-
- public void cacheBaseStatistics(BaseStatistics statistics, String identifyer) {
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/9d222ca1/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/DenseValueSchema.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/DenseValueSchema.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/DenseValueSchema.java
deleted file mode 100644
index ecce33d..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/DenseValueSchema.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.postpass;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.types.Value;
-
-/**
- * Class encapsulating a schema map (int column position -> column type) and a reference counter.
- */
-public class DenseValueSchema extends AbstractSchema<Class<? extends Value>> {
-
- private final List<Class<? extends Value>> schema;
-
- private int numFields = -1;
-
-
- public DenseValueSchema() {
- this.schema = new ArrayList<Class<? extends Value>>();
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void addType(int pos, Class<? extends Value> type) throws ConflictingFieldTypeInfoException {
- if (pos == schema.size()) {
- // right at the end, most common case
- schema.add(type);
- } else if (pos < schema.size()) {
- // in the middle, check for existing conflicts
- Class<? extends Value> previous = schema.get(pos);
- if (previous == null) {
- schema.set(pos, type);
- } else if (previous != type) {
- throw new ConflictingFieldTypeInfoException(pos, previous, type);
- }
- } else {
- // grow to the end
- for (int i = schema.size(); i <= pos; i++) {
- schema.add(null);
- }
- schema.set(pos, type);
- }
- }
-
- @Override
- public Class<? extends Value> getType(int field) {
- return schema.get(field);
- }
-
- @Override
- public Iterator<Map.Entry<Integer, Class<? extends Value>>> iterator() {
- final Iterator<Class<? extends Value>> iter = schema.iterator();
- return new Iterator<Map.Entry<Integer,Class<? extends Value>>>() {
-
- private int pos = 0;
-
- @Override
- public boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public java.util.Map.Entry<Integer, Class<? extends Value>> next() {
- return new Entry(pos++, iter.next());
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- public int getNumFields() {
- return numFields;
- }
-
-
- public void setNumFields(int numFields) {
- this.numFields = numFields;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- return schema.hashCode() ^ getNumConnectionsThatContributed();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof DenseValueSchema) {
- DenseValueSchema other = (DenseValueSchema) obj;
- return schema.equals(other.schema) &&
- getNumConnectionsThatContributed() == other.getNumConnectionsThatContributed();
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "<" + getNumConnectionsThatContributed() + "> : " + schema.toString();
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static final class Entry implements Map.Entry<Integer, Class<? extends Value>> {
-
- private Integer key;
- private Class<? extends Value> value;
-
-
- public Entry(Integer key, Class<? extends Value> value) {
- this.key = key;
- this.value = value;
- }
-
- @Override
- public Integer getKey() {
- return this.key;
- }
-
- @Override
- public Class<? extends Value> getValue() {
- return this.value;
- }
-
- @Override
- public Class<? extends Value> setValue(Class<? extends Value> value) {
- throw new UnsupportedOperationException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/9d222ca1/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
index 3ba3446..b9f6bfa 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/MissingFieldTypeInfoException.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.postpass;
public final class MissingFieldTypeInfoException extends Exception {
http://git-wip-us.apache.org/repos/asf/flink/blob/9d222ca1/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
index c4b95d1..e14888e 100644
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
+++ b/flink-compiler/src/main/java/org/apache/flink/optimizer/postpass/SparseKeySchema.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.optimizer.postpass;
import java.util.HashMap;
[09/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
new file mode 100644
index 0000000..c2e81a8
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/InterestingPropertyVisitor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+/**
+ * Visitor that computes the interesting properties for each node in the optimizer DAG. On its recursive
+ * depth-first descend, it propagates all interesting properties top-down.
+ */
+public class InterestingPropertyVisitor implements Visitor<OptimizerNode> {
+
+ private CostEstimator estimator; // the cost estimator for maximal costs of an interesting property
+
+ /**
+ * Creates a new visitor that computes the interesting properties for all nodes in the plan.
+ * It uses the given cost estimator used to compute the maximal costs for an interesting property.
+ *
+ * @param estimator
+ * The cost estimator to estimate the maximal costs for interesting properties.
+ */
+ public InterestingPropertyVisitor(CostEstimator estimator) {
+ this.estimator = estimator;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode node) {
+ // The interesting properties must be computed on the descend. In case a node has multiple outputs,
+ // that computation must happen during the last descend.
+
+ if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
+ node.computeUnionOfInterestingPropertiesFromSuccessors();
+ node.computeInterestingPropertiesForInputs(this.estimator);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
new file mode 100644
index 0000000..58aa3c1
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/PlanFinalizer.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.BinaryUnionPlanNode;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.IterationPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This visitor traverses the selected execution plan and finalizes it:
+ *
+ * <ul>
+ * <li>The graph of nodes is double-linked (links from child to parent are inserted).</li>
+ * <li>If unions join static and dynamic paths, the cache is marked as a memory consumer.</li>
+ * <li>Relative memory fractions are assigned to all nodes.</li>
+ * <li>All nodes are collected into a set.</li>
+ * </ul>
+ */
+public class PlanFinalizer implements Visitor<PlanNode> {
+
+ private final Set<PlanNode> allNodes; // a set of all nodes in the optimizer plan
+
+ private final List<SourcePlanNode> sources; // all data source nodes in the optimizer plan
+
+ private final List<SinkPlanNode> sinks; // all data sink nodes in the optimizer plan
+
+ private final Deque<IterationPlanNode> stackOfIterationNodes;
+
+ private int memoryConsumerWeights; // a counter of all memory consumers
+
+ /**
+ * Creates a new plan finalizer.
+ */
+ public PlanFinalizer() {
+ this.allNodes = new HashSet<PlanNode>();
+ this.sources = new ArrayList<SourcePlanNode>();
+ this.sinks = new ArrayList<SinkPlanNode>();
+ this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
+ }
+
+ public OptimizedPlan createFinalPlan(List<SinkPlanNode> sinks, String jobName, Plan originalPlan) {
+ this.memoryConsumerWeights = 0;
+
+ // traverse the graph
+ for (SinkPlanNode node : sinks) {
+ node.accept(this);
+ }
+
+ // assign the memory to each node
+ if (this.memoryConsumerWeights > 0) {
+ for (PlanNode node : this.allNodes) {
+ // assign memory to the driver strategy of the node
+ final int consumerWeight = node.getMemoryConsumerWeight();
+ if (consumerWeight > 0) {
+ final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights;
+ node.setRelativeMemoryPerSubtask(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " +
+ node.getProgramOperator().getName() + ".");
+ }
+ }
+
+ // assign memory to the local and global strategies of the channels
+ for (Channel c : node.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ final double relativeMem = 1.0 / this.memoryConsumerWeights;
+ c.setRelativeMemoryLocalStrategy(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each local strategy " +
+ "instance of " + c + ".");
+ }
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ final double relativeMem = 1.0/ this.memoryConsumerWeights;
+ c.setRelativeTempMemory(relativeMem);
+ if (Optimizer.LOG.isDebugEnabled()) {
+ Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each instance of the temp " +
+ "table for " + c + ".");
+ }
+ }
+ }
+ }
+ }
+ return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
+ }
+
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ // if we come here again, prevent a further descend
+ if (!this.allNodes.add(visitable)) {
+ return false;
+ }
+
+ if (visitable instanceof SinkPlanNode) {
+ this.sinks.add((SinkPlanNode) visitable);
+ }
+ else if (visitable instanceof SourcePlanNode) {
+ this.sources.add((SourcePlanNode) visitable);
+ }
+ else if (visitable instanceof BinaryUnionPlanNode) {
+ BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
+ if (unionNode.unionsStaticAndDynamicPath()) {
+ unionNode.setDriverStrategy(DriverStrategy.UNION_WITH_CACHED);
+ }
+ }
+ else if (visitable instanceof BulkPartialSolutionPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof BulkIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof WorksetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final WorksetPlanNode wspn = (WorksetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ wspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+ else if (visitable instanceof SolutionSetPlanNode) {
+ // tell the partial solution about the iteration node that contains it
+ final SolutionSetPlanNode sspn = (SolutionSetPlanNode) visitable;
+ final IterationPlanNode iteration = this.stackOfIterationNodes.peekLast();
+
+ // sanity check!
+ if (iteration == null || !(iteration instanceof WorksetIterationPlanNode)) {
+ throw new CompilerException("Bug: Error finalizing the plan. " +
+ "Cannot associate the node for a partial solutions with its containing iteration.");
+ }
+ sspn.setContainingIterationNode((WorksetIterationPlanNode) iteration);
+ }
+
+ // double-connect the connections. previously, only parents knew their children, because
+ // one child candidate could have been referenced by multiple parents.
+ for (Channel conn : visitable.getInputs()) {
+ conn.setTarget(visitable);
+ conn.getSource().addOutgoingChannel(conn);
+ }
+
+ for (Channel c : visitable.getBroadcastInputs()) {
+ c.setTarget(visitable);
+ c.getSource().addOutgoingChannel(c);
+ }
+
+ // count the memory consumption
+ this.memoryConsumerWeights += visitable.getMemoryConsumerWeight();
+ for (Channel c : visitable.getInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+ for (Channel c : visitable.getBroadcastInputs()) {
+ if (c.getLocalStrategy().dams()) {
+ this.memoryConsumerWeights++;
+ }
+ if (c.getTempMode() != TempMode.NONE) {
+ this.memoryConsumerWeights++;
+ }
+ }
+
+ // pass the visitor to the iteraton's step function
+ if (visitable instanceof IterationPlanNode) {
+ // push the iteration node onto the stack
+ final IterationPlanNode iterNode = (IterationPlanNode) visitable;
+ this.stackOfIterationNodes.addLast(iterNode);
+
+ // recurse
+ ((IterationPlanNode) visitable).acceptForStepFunction(this);
+
+ // pop the iteration node from the stack
+ this.stackOfIterationNodes.removeLast();
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
new file mode 100644
index 0000000..c0dc4dd
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StaticDynamicPathIdentifier.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.IterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.util.Visitor;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A traversal that goes over the program data flow of an iteration and makes the nodes
+ * that depend on the partial solution (the data set recomputed in each iteration) as "dynamic"
+ * and the other nodes as "static".
+ */
+public class StaticDynamicPathIdentifier implements Visitor<OptimizerNode> {
+
+ private final Set<OptimizerNode> seenBefore = new HashSet<OptimizerNode>();
+
+ private final int costWeight;
+
+ public StaticDynamicPathIdentifier(int costWeight) {
+ this.costWeight = costWeight;
+ }
+
+ @Override
+ public boolean preVisit(OptimizerNode visitable) {
+ return this.seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(OptimizerNode visitable) {
+ visitable.identifyDynamicPath(this.costWeight);
+
+ // check that there is no nested iteration on the dynamic path
+ if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
+ throw new CompilerException("Nested iterations are currently not supported.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
new file mode 100644
index 0000000..d359490
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/StepFunctionValidator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.traversals;
+
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.util.Visitor;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A traversal that checks if the Workset of a delta iteration is used in the data flow
+ * of its step function.
+ */
+public class StepFunctionValidator implements Visitor<Operator<?>> {
+
+ private final Set<Operator<?>> seenBefore = new HashSet<Operator<?>>();
+
+ private boolean foundWorkset;
+
+ @Override
+ public boolean preVisit(Operator<?> visitable) {
+ if (visitable instanceof DeltaIterationBase.WorksetPlaceHolder) {
+ foundWorkset = true;
+ }
+
+ return (!foundWorkset) && seenBefore.add(visitable);
+ }
+
+ @Override
+ public void postVisit(Operator<?> visitable) {}
+
+ public boolean hasFoundWorkset() {
+ return foundWorkset;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/package-info.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
new file mode 100644
index 0000000..cd8766c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/traversals/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the various traversals over the program plan and the
+ * optimizer DAG (directed acyclic graph) that are made in the course of
+ * the optimization.
+ *
+ * The traversals are mostly implemented as a {@link org.apache.flink.util.Visitor} that
+ * traversed the program flow.
+ */
+package org.apache.flink.optimizer.traversals;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
new file mode 100644
index 0000000..5110849
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpBinaryUdfOp.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.util.List;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.functions.util.NoOpFunction;
+import org.apache.flink.api.common.operators.BinaryOperatorInformation;
+import org.apache.flink.api.common.operators.DualInputOperator;
+import org.apache.flink.api.common.operators.RecordOperator;
+import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.types.Key;
+
+
+public class NoOpBinaryUdfOp<OUT> extends DualInputOperator<OUT, OUT, OUT, NoOpFunction> implements RecordOperator {
+
+ public NoOpBinaryUdfOp(TypeInformation<OUT> type) {
+ super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), new BinaryOperatorInformation<OUT, OUT, OUT>(type, type, type), "NoContract");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Class<? extends Key<?>>[] getKeyClasses() {
+ return (Class<? extends Key<?>>[]) new Class[0];
+ }
+
+ @Override
+ protected List<OUT> executeOnCollections(List<OUT> inputData1, List<OUT> inputData2, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {
+ throw new UnsupportedOperationException();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
new file mode 100644
index 0000000..cc4a4d6
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/NoOpUnaryUdfOp.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.util;
+
+import java.util.List;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.functions.util.NoOpFunction;
+import org.apache.flink.api.common.operators.RecordOperator;
+import org.apache.flink.api.common.operators.SingleInputOperator;
+import org.apache.flink.api.common.operators.UnaryOperatorInformation;
+import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.types.Key;
+
+
+public class NoOpUnaryUdfOp<OUT> extends SingleInputOperator<OUT, OUT, NoOpFunction> implements RecordOperator {
+
+ @SuppressWarnings("rawtypes")
+ public static final NoOpUnaryUdfOp INSTANCE = new NoOpUnaryUdfOp();
+
+ private NoOpUnaryUdfOp() {
+ // pass null here because we override getOutputType to return type
+ // of input operator
+ super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), null, "");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Class<? extends Key<?>>[] getKeyClasses() {
+ return (Class<? extends Key<?>>[]) new Class[0];
+ }
+
+ @Override
+ public UnaryOperatorInformation<OUT, OUT> getOperatorInfo() {
+ TypeInformation<OUT> previousOut = input.getOperatorInfo().getOutputType();
+ return new UnaryOperatorInformation<OUT, OUT>(previousOut, previousOut);
+ }
+
+ @Override
+ protected List<OUT> executeOnCollections(List<OUT> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {
+ return inputData;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/Utils.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/Utils.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/Utils.java
new file mode 100644
index 0000000..d8f33a2
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/util/Utils.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.util;
+
+import java.util.Arrays;
+
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+
+
+/**
+ *
+ */
+public class Utils
+{
+ public static final FieldList createOrderedFromSet(FieldSet set) {
+ if (set instanceof FieldList) {
+ return (FieldList) set;
+ } else {
+ final int[] cols = set.toArray();
+ Arrays.sort(cols);
+ return new FieldList(cols);
+ }
+ }
+
+ public static final Ordering createOrdering(FieldList fields, boolean[] directions) {
+ final Ordering o = new Ordering();
+ for (int i = 0; i < fields.size(); i++) {
+ o.appendOrdering(fields.get(i), null, directions == null || directions[i] ? Order.ASCENDING : Order.DESCENDING);
+ }
+ return o;
+ }
+
+ public static final Ordering createOrdering(FieldList fields) {
+ final Ordering o = new Ordering();
+ for (int i = 0; i < fields.size(); i++) {
+ o.appendOrdering(fields.get(i), null, Order.ANY);
+ }
+ return o;
+ }
+
+ public static boolean[] getDirections(Ordering o, int numFields) {
+ final boolean[] dirs = o.getFieldSortDirections();
+ if (dirs.length == numFields) {
+ return dirs;
+ } else if (dirs.length > numFields) {
+ final boolean[] subSet = new boolean[numFields];
+ System.arraycopy(dirs, 0, subSet, 0, numFields);
+ return subSet;
+ } else {
+ throw new CompilerException();
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * No instantiation.
+ */
+ private Utils() {}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
new file mode 100644
index 0000000..1e4bafb
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/AdditionalOperatorsTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.record.operators.CrossOperator;
+import org.apache.flink.api.java.record.operators.CrossWithLargeOperator;
+import org.apache.flink.api.java.record.operators.CrossWithSmallOperator;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.util.DummyCrossStub;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+/**
+* Tests that validate optimizer choices when using operators that are requesting certain specific execution
+* strategies.
+*/
+@SuppressWarnings({"serial", "deprecation"})
+public class AdditionalOperatorsTest extends CompilerTestBase {
+
+ @Test
+ public void testCrossWithSmall() {
+ // construct the plan
+ FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
+ FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
+
+ CrossOperator cross = CrossWithSmallOperator.builder(new DummyCrossStub())
+ .input1(source1).input2(source2)
+ .name("Cross").build();
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
+
+ Plan plan = new Plan(sink);
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+
+ try {
+ OptimizedPlan oPlan = compileNoStats(plan);
+ OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
+
+ DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
+ Channel in1 = crossPlanNode.getInput1();
+ Channel in2 = crossPlanNode.getInput2();
+
+ assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
+ assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ }
+ }
+
+ @Test
+ public void testCrossWithLarge() {
+ // construct the plan
+ FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 1");
+ FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE, "Source 2");
+
+ CrossOperator cross= CrossWithLargeOperator.builder(new DummyCrossStub())
+ .input1(source1).input2(source2)
+ .name("Cross").build();
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cross, "Sink");
+
+ Plan plan = new Plan(sink);
+ plan.setDefaultParallelism(DEFAULT_PARALLELISM);
+
+
+ try {
+ OptimizedPlan oPlan = compileNoStats(plan);
+ OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
+
+ DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
+ Channel in1 = crossPlanNode.getInput1();
+ Channel in2 = crossPlanNode.getInput2();
+
+ assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
+ } catch(CompilerException ce) {
+ ce.printStackTrace();
+ fail("The pact compiler is unable to compile this plan correctly.");
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
new file mode 100644
index 0000000..916aa27
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/BranchingPlansCompilerTest.java
@@ -0,0 +1,1039 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
+import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
+import org.apache.flink.util.Collector;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichJoinFunction;
+import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint;
+import org.apache.flink.api.java.record.operators.BulkIteration;
+import org.apache.flink.api.java.record.operators.CoGroupOperator;
+import org.apache.flink.api.java.record.operators.CrossOperator;
+import org.apache.flink.api.java.record.operators.DeltaIteration;
+import org.apache.flink.api.java.record.operators.FileDataSink;
+import org.apache.flink.api.java.record.operators.FileDataSource;
+import org.apache.flink.api.java.record.operators.JoinOperator;
+import org.apache.flink.api.java.record.operators.MapOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
+import org.apache.flink.optimizer.testfunctions.IdentityGroupReducer;
+import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
+import org.apache.flink.optimizer.util.DummyCoGroupStub;
+import org.apache.flink.optimizer.util.DummyCrossStub;
+import org.apache.flink.optimizer.util.DummyInputFormat;
+import org.apache.flink.optimizer.util.DummyMatchStub;
+import org.apache.flink.optimizer.util.DummyNonPreservingMatchStub;
+import org.apache.flink.optimizer.util.DummyOutputFormat;
+import org.apache.flink.optimizer.util.IdentityMap;
+import org.apache.flink.optimizer.util.IdentityReduce;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+
+@SuppressWarnings({"serial", "deprecation"})
+public class BranchingPlansCompilerTest extends CompilerTestBase {
+
+
+ @Test
+ public void testCostComputationWithMultipleDataSinks() {
+ final int SINKS = 5;
+
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Long> source = env.generateSequence(1, 10000);
+
+ DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedC = source.map(new IdentityMapper<Long>());
+
+ for (int sink = 0; sink < SINKS; sink++) {
+ mappedA.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
+ }
+
+ Plan plan = env.createProgramPlan("Plans With Multiple Data Sinks");
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+
+ /**
+ *
+ * <pre>
+ * (SRC A)
+ * |
+ * (MAP A)
+ * / \
+ * (MAP B) (MAP C)
+ * / / \
+ * (SINK A) (SINK B) (SINK C)
+ * </pre>
+ */
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testBranchingWithMultipleDataSinks2() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Long> source = env.generateSequence(1, 10000);
+
+ DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
+ DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());
+
+ mappedB.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
+ mappedC.output(new DiscardingOutputFormat<Long>());
+
+ Plan plan = env.createProgramPlan();
+ Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // ---------- check the optimizer plan ----------
+
+ // number of sinks
+ assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());
+
+ // remove matching sinks to check relation
+ for (SinkPlanNode sink : oPlan.getDataSinks()) {
+ assertTrue(sinks.remove(sink.getProgramOperator()));
+ }
+ assertTrue(sinks.isEmpty());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+
+ /**
+ * <pre>
+ * SINK
+ * |
+ * COGROUP
+ * +---/ \----+
+ * / \
+ * / MATCH10
+ * / | \
+ * / | MATCH9
+ * MATCH5 | | \
+ * | \ | | MATCH8
+ * | MATCH4 | | | \
+ * | | \ | | | MATCH7
+ * | | MATCH3 | | | | \
+ * | | | \ | | | | MATCH6
+ * | | | MATCH2 | | | | | |
+ * | | | | \ +--+--+--+--+--+
+ * | | | | MATCH1 MAP
+ * \ | | | | | /-----------/
+ * (DATA SOURCE ONE)
+ * </pre>
+ */
+ @Test
+ public void testBranchingSourceMultipleTimes() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Tuple2<Long, Long>> source = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> joined1 = source.join(source).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined2 = source.join(joined1).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined3 = source.join(joined2).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined4 = source.join(joined3).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined5 = source.join(joined4).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> mapped = source.map(
+ new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Tuple2<Long, Long> value) {
+ return null;
+ }
+ });
+
+ DataSet<Tuple2<Long, Long>> joined6 = mapped.join(mapped).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined7 = mapped.join(joined6).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined8 = mapped.join(joined7).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined9 = mapped.join(joined8).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined10 = mapped.join(joined9).where(0).equalTo(0)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+
+ joined5.coGroup(joined10)
+ .where(1).equalTo(1)
+ .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
+
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
+
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileNoStats(plan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ /**
+ *
+ * <pre>
+
+ * (SINK A)
+ * | (SINK B) (SINK C)
+ * CROSS / /
+ * / \ | +------+
+ * / \ | /
+ * REDUCE MATCH2
+ * | +---/ \
+ * \ / |
+ * MAP |
+ * | |
+ * COGROUP MATCH1
+ * / \ / \
+ * (SRC A) (SRC B) (SRC C)
+ * </pre>
+ */
+ @Test
+ public void testBranchingWithMultipleDataSinks() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Tuple2<Long, Long>> sourceA = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> sourceB = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> sourceC = env.generateSequence(1, 10000000)
+ .map(new Duplicator<Long>());
+
+ DataSet<Tuple2<Long, Long>> mapped = sourceA.coGroup(sourceB)
+ .where(0).equalTo(1)
+ .with(new CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
+ @Override
+ public void coGroup(Iterable<Tuple2<Long, Long>> first,
+ Iterable<Tuple2<Long, Long>> second,
+ Collector<Tuple2<Long, Long>> out) {
+ }
+ })
+ .map(new IdentityMapper<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined = sourceB.join(sourceC)
+ .where(0).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> joined2 = mapped.join(joined)
+ .where(1).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
+
+ DataSet<Tuple2<Long, Long>> reduced = mapped
+ .groupBy(1)
+ .reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>());
+
+ reduced.cross(joined2)
+ .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
+
+ joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+ joined2.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
+
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileNoStats(plan);
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testBranchEachContractType() {
+ try {
+ // construct the plan
+ FileDataSource sourceA = new FileDataSource(new DummyInputFormat(), "file:///test/file1", "Source A");
+ FileDataSource sourceB = new FileDataSource(new DummyInputFormat(), "file:///test/file2", "Source B");
+ FileDataSource sourceC = new FileDataSource(new DummyInputFormat(), "file:///test/file3", "Source C");
+
+ MapOperator map1 = MapOperator.builder(new IdentityMap()).input(sourceA).name("Map 1").build();
+
+ ReduceOperator reduce1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(map1)
+ .name("Reduce 1")
+ .build();
+
+ JoinOperator match1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
+ .input1(sourceB, sourceB, sourceC)
+ .input2(sourceC)
+ .name("Match 1")
+ .build();
+ ;
+ CoGroupOperator cogroup1 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(sourceA)
+ .input2(sourceB)
+ .name("CoGroup 1")
+ .build();
+
+ CrossOperator cross1 = CrossOperator.builder(new DummyCrossStub())
+ .input1(reduce1)
+ .input2(cogroup1)
+ .name("Cross 1")
+ .build();
+
+
+ CoGroupOperator cogroup2 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(cross1)
+ .input2(cross1)
+ .name("CoGroup 2")
+ .build();
+
+ CoGroupOperator cogroup3 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(map1)
+ .input2(match1)
+ .name("CoGroup 3")
+ .build();
+
+
+ MapOperator map2 = MapOperator.builder(new IdentityMap()).input(cogroup3).name("Map 2").build();
+
+ CoGroupOperator cogroup4 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(map2)
+ .input2(match1)
+ .name("CoGroup 4")
+ .build();
+
+ CoGroupOperator cogroup5 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(cogroup2)
+ .input2(cogroup1)
+ .name("CoGroup 5")
+ .build();
+
+ CoGroupOperator cogroup6 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(reduce1)
+ .input2(cogroup4)
+ .name("CoGroup 6")
+ .build();
+
+ CoGroupOperator cogroup7 = CoGroupOperator.builder(new DummyCoGroupStub(), IntValue.class, 0,0)
+ .input1(cogroup5)
+ .input2(cogroup6)
+ .name("CoGroup 7")
+ .build();
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, cogroup7);
+ sink.addInput(sourceA);
+ sink.addInput(cogroup3);
+ sink.addInput(cogroup4);
+ sink.addInput(cogroup1);
+
+ // return the PACT plan
+ Plan plan = new Plan(sink, "Branching of each contract type");
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+
+ //Compile plan to verify that no error is thrown
+ jobGen.compileJobGraph(oPlan);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void testBranchingUnion() {
+ try {
+ // construct the plan
+ FileDataSource source1 = new FileDataSource(new DummyInputFormat(), IN_FILE);
+ FileDataSource source2 = new FileDataSource(new DummyInputFormat(), IN_FILE);
+
+ JoinOperator mat1 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
+ .input1(source1)
+ .input2(source2)
+ .name("Match 1")
+ .build();
+
+ MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
+
+ ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(ma1)
+ .name("Reduce 1")
+ .build();
+
+ ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
+ .input(mat1)
+ .name("Reduce 2")
+ .build();
+
+ MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
+
+ MapOperator ma3 = MapOperator.builder(new IdentityMap()).input(ma2).name("Map 3").build();
+
+ @SuppressWarnings("unchecked")
+ JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
+ .input1(r1, r2, ma2, ma3)
+ .input2(ma2)
+ .name("Match 2")
+ .build();
+ mat2.setParameter(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_MERGE);
+
+ FileDataSink sink = new FileDataSink(new DummyOutputFormat(), OUT_FILE, mat2);
+
+
+ // return the PACT plan
+ Plan plan = new Plan(sink, "Branching Union");
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+
+ //Compile plan to verify that no error is thrown
+ jobGen.compileJobGraph(oPlan);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ *
+ * <pre>
+ * (SRC A)
+ * / \
+ * (SINK A) (SINK B)
+ * </pre>
+ */
+ @Test
+ public void testBranchingWithMultipleDataSinksSmall() {
+ try {
+ // construct the plan
+ final String out1Path = "file:///test/1";
+ final String out2Path = "file:///test/2";
+
+ FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
+
+ FileDataSink sinkA = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA);
+ FileDataSink sinkB = new FileDataSink(DummyOutputFormat.class, out2Path, sourceA);
+
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sinkA);
+ sinks.add(sinkB);
+
+ // return the PACT plan
+ Plan plan = new Plan(sinks, "Plans With Multiple Data Sinks");
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ // ---------- check the optimizer plan ----------
+
+ // number of sinks
+ Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
+
+ // sinks contain all sink paths
+ Set<String> allSinks = new HashSet<String>();
+ allSinks.add(out1Path);
+ allSinks.add(out2Path);
+
+ for (SinkPlanNode n : oPlan.getDataSinks()) {
+ String path = ((FileDataSink) n.getSinkNode().getOperator()).getFilePath();
+ Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
+ }
+
+ // ---------- compile plan to nephele job graph to verify that no error is thrown ----------
+
+ JobGraphGenerator jobGen = new JobGraphGenerator();
+ jobGen.compileJobGraph(oPlan);
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ *
+ * <pre>
+ * (SINK 3) (SINK 1) (SINK 2) (SINK 4)
+ * \ / \ /
+ * (SRC A) (SRC B)
+ * </pre>
+ *
+ * NOTE: this case is currently not caught by the compiler. we should enable the test once it is caught.
+ */
+ @Test
+ public void testBranchingDisjointPlan() {
+ // construct the plan
+ final String out1Path = "file:///test/1";
+ final String out2Path = "file:///test/2";
+ final String out3Path = "file:///test/3";
+ final String out4Path = "file:///test/4";
+
+ FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE);
+ FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE);
+
+ FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, out1Path, sourceA, "1");
+ FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, out2Path, sourceB, "2");
+ FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, out3Path, sourceA, "3");
+ FileDataSink sink4 = new FileDataSink(DummyOutputFormat.class, out4Path, sourceB, "4");
+
+
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sink1);
+ sinks.add(sink2);
+ sinks.add(sink3);
+ sinks.add(sink4);
+
+ // return the PACT plan
+ Plan plan = new Plan(sinks, "Disjoint plan with multiple data sinks and branches");
+ compileNoStats(plan);
+ }
+
+ @Test
+ public void testBranchAfterIteration() {
+ FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
+
+ BulkIteration iteration = new BulkIteration("Loop");
+ iteration.setInput(sourceA);
+ iteration.setMaximumNumberOfIterations(10);
+
+ MapOperator mapper = MapOperator.builder(IdentityMap.class).name("Mapper").input(iteration.getPartialSolution()).build();
+ iteration.setNextPartialSolution(mapper);
+
+ FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 1");
+
+ MapOperator postMap = MapOperator.builder(IdentityMap.class).name("Post Iteration Mapper")
+ .input(iteration).build();
+
+ FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink 2");
+
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sink1);
+ sinks.add(sink2);
+
+ Plan plan = new Plan(sinks);
+
+ try {
+ compileNoStats(plan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBranchBeforeIteration() {
+ FileDataSource source1 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
+ FileDataSource source2 = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
+
+ BulkIteration iteration = new BulkIteration("Loop");
+ iteration.setInput(source2);
+ iteration.setMaximumNumberOfIterations(10);
+
+ MapOperator inMap = MapOperator.builder(new IdentityMap())
+ .input(source1)
+ .name("In Iteration Map")
+ .setBroadcastVariable("BC", iteration.getPartialSolution())
+ .build();
+
+ iteration.setNextPartialSolution(inMap);
+
+ MapOperator postMap = MapOperator.builder(new IdentityMap())
+ .input(source1)
+ .name("Post Iteration Map")
+ .setBroadcastVariable("BC", iteration)
+ .build();
+
+ FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink");
+
+ Plan plan = new Plan(sink);
+
+ try {
+ compileNoStats(plan);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ * Test to ensure that sourceA is inside as well as outside of the iteration the same
+ * node.
+ *
+ * <pre>
+ * (SRC A) (SRC B)
+ * / \ / \
+ * (SINK 1) (ITERATION) | (SINK 2)
+ * / \ /
+ * (SINK 3) (CROSS => NEXT PARTIAL SOLUTION)
+ * </pre>
+ */
+ @Test
+ public void testClosure() {
+ FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
+ FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
+
+ FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
+ FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceB, "Sink 2");
+
+ BulkIteration iteration = new BulkIteration("Loop");
+ iteration.setInput(sourceA);
+ iteration.setMaximumNumberOfIterations(10);
+
+ CrossOperator stepFunction = CrossOperator.builder(DummyCrossStub.class).name("StepFunction").
+ input1(iteration.getPartialSolution()).
+ input2(sourceB).
+ build();
+
+ iteration.setNextPartialSolution(stepFunction);
+
+ FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");
+
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sink1);
+ sinks.add(sink2);
+ sinks.add(sink3);
+
+ Plan plan = new Plan(sinks);
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ * <pre>
+ * (SRC A) (SRC B) (SRC C)
+ * / \ / / \
+ * (SINK 1) (DELTA ITERATION) | (SINK 2)
+ * / | \ /
+ * (SINK 3) | (CROSS => NEXT WORKSET)
+ * | |
+ * (JOIN => SOLUTION SET DELTA)
+ * </pre>
+ */
+ @Test
+ public void testClosureDeltaIteration() {
+ FileDataSource sourceA = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 1");
+ FileDataSource sourceB = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 2");
+ FileDataSource sourceC = new FileDataSource(DummyInputFormat.class, IN_FILE, "Source 3");
+
+ FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceA, "Sink 1");
+ FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, sourceC, "Sink 2");
+
+ DeltaIteration iteration = new DeltaIteration(0, "Loop");
+ iteration.setInitialSolutionSet(sourceA);
+ iteration.setInitialWorkset(sourceB);
+ iteration.setMaximumNumberOfIterations(10);
+
+ CrossOperator nextWorkset = CrossOperator.builder(DummyCrossStub.class).name("Next workset").
+ input1(iteration.getWorkset()).
+ input2(sourceC).
+ build();
+
+ JoinOperator solutionSetDelta = JoinOperator.builder(DummyMatchStub.class, LongValue.class,0,0).
+ name("Next solution set.").
+ input1(nextWorkset).
+ input2(iteration.getSolutionSet()).
+ build();
+
+ iteration.setNextWorkset(nextWorkset);
+ iteration.setSolutionSetDelta(solutionSetDelta);
+
+ FileDataSink sink3 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 3");
+
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sink1);
+ sinks.add(sink2);
+ sinks.add(sink3);
+
+ Plan plan = new Plan(sinks);
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ * <pre>
+ * +----Iteration-------+
+ * | |
+ * /---------< >---------join-----< >---sink
+ * / (Solution)| / |
+ * / | / |
+ * /--map-------< >----\ / /--|
+ * / (Workset)| \ / / |
+ * src-map | join------/ |
+ * \ | / |
+ * \ +-----/--------------+
+ * \ /
+ * \--reduce-------/
+ * </pre>
+ */
+ @Test
+ public void testDeltaIterationWithStaticInput() {
+ FileDataSource source = new FileDataSource(DummyInputFormat.class, IN_FILE, "source");
+
+ MapOperator mappedSource = MapOperator.builder(IdentityMap.class).
+ input(source).
+ name("Identity mapped source").
+ build();
+
+ ReduceOperator reducedSource = ReduceOperator.builder(IdentityReduce.class).
+ input(source).
+ name("Identity reduce source").
+ build();
+
+ DeltaIteration iteration = new DeltaIteration(0,"Loop");
+ iteration.setMaximumNumberOfIterations(10);
+ iteration.setInitialSolutionSet(source);
+ iteration.setInitialWorkset(mappedSource);
+
+ JoinOperator nextWorkset = JoinOperator.builder(DummyNonPreservingMatchStub.class, IntValue.class, 0,0).
+ input1(iteration.getWorkset()).
+ input2(reducedSource).
+ name("Next work set").
+ build();
+
+ JoinOperator solutionSetDelta = JoinOperator.builder(DummyNonPreservingMatchStub.class, IntValue.class, 0,
+ 0).
+ input1(iteration.getSolutionSet()).
+ input2(nextWorkset).
+ name("Solution set delta").
+ build();
+
+ iteration.setNextWorkset(nextWorkset);
+ iteration.setSolutionSetDelta(solutionSetDelta);
+
+ FileDataSink sink = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Iteration sink");
+ List<FileDataSink> sinks = new ArrayList<FileDataSink>();
+ sinks.add(sink);
+
+ Plan plan = new Plan(sinks);
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ /**
+ * <pre>
+ * +---------Iteration-------+
+ * | |
+ * /--map--< >----\ |
+ * / | \ /-------< >---sink
+ * src-map | join------/ |
+ * \ | / |
+ * \ +-----/-------------------+
+ * \ /
+ * \--reduce--/
+ * </pre>
+ */
+ @Test
+ public void testIterationWithStaticInput() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<Long> source = env.generateSequence(1, 1000000);
+
+ DataSet<Long> mapped = source.map(new IdentityMapper<Long>());
+
+ DataSet<Long> reduced = source.groupBy(new IdentityKeyExtractor<Long>()).reduce(new SelectOneReducer<Long>());
+
+ IterativeDataSet<Long> iteration = mapped.iterate(10);
+ iteration.closeWith(
+ iteration.join(reduced)
+ .where(new IdentityKeyExtractor<Long>())
+ .equalTo(new IdentityKeyExtractor<Long>())
+ .with(new DummyFlatJoinFunction<Long>()))
+ .output(new DiscardingOutputFormat<Long>());
+
+ compileNoStats(env.createProgramPlan());
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBranchingBroadcastVariable() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<String> input1 = env.readTextFile(IN_FILE).name("source1");
+ DataSet<String> input2 = env.readTextFile(IN_FILE).name("source2");
+ DataSet<String> input3 = env.readTextFile(IN_FILE).name("source3");
+
+ DataSet<String> result1 = input1
+ .map(new IdentityMapper<String>())
+ .reduceGroup(new Top1GroupReducer<String>())
+ .withBroadcastSet(input3, "bc");
+
+ DataSet<String> result2 = input2
+ .map(new IdentityMapper<String>())
+ .reduceGroup(new Top1GroupReducer<String>())
+ .withBroadcastSet(input3, "bc");
+
+ result1.join(result2)
+ .where(new IdentityKeyExtractor<String>())
+ .equalTo(new IdentityKeyExtractor<String>())
+ .with(new RichJoinFunction<String, String, String>() {
+ @Override
+ public String join(String first, String second) {
+ return null;
+ }
+ })
+ .withBroadcastSet(input3, "bc1")
+ .withBroadcastSet(input1, "bc2")
+ .withBroadcastSet(result1, "bc3")
+ .print();
+
+ Plan plan = env.createProgramPlan();
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBCVariableClosure() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
+
+ DataSet<String> reduced = input
+ .map(new IdentityMapper<String>())
+ .reduceGroup(new Top1GroupReducer<String>());
+
+
+ DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
+
+
+ IterativeDataSet<String> iteration = initialSolution.iterate(100);
+
+ iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
+ .print();
+
+ Plan plan = env.createProgramPlan();
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testMultipleIterations() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
+
+ DataSet<String> reduced = input
+ .map(new IdentityMapper<String>())
+ .reduceGroup(new Top1GroupReducer<String>());
+
+ IterativeDataSet<String> iteration1 = input.iterate(100);
+ IterativeDataSet<String> iteration2 = input.iterate(20);
+ IterativeDataSet<String> iteration3 = input.iterate(17);
+
+ iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")).print();
+ iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")).print();
+ iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")).print();
+
+ Plan plan = env.createProgramPlan();
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testMultipleIterationsWithClosueBCVars() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
+
+ IterativeDataSet<String> iteration1 = input.iterate(100);
+ IterativeDataSet<String> iteration2 = input.iterate(20);
+ IterativeDataSet<String> iteration3 = input.iterate(17);
+
+
+ iteration1.closeWith(iteration1.map(new IdentityMapper<String>())).print();
+ iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())).print();
+ iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())).print();
+
+ Plan plan = env.createProgramPlan();
+
+ try{
+ compileNoStats(plan);
+ }catch(Exception e){
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBranchesOnlyInBCVariables1() {
+ try{
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<Long> input = env.generateSequence(1, 10);
+ DataSet<Long> bc_input = env.generateSequence(1, 10);
+
+ input
+ .map(new IdentityMapper<Long>()).withBroadcastSet(bc_input, "name1")
+ .map(new IdentityMapper<Long>()).withBroadcastSet(bc_input, "name2")
+ .print();
+
+ Plan plan = env.createProgramPlan();
+ compileNoStats(plan);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBranchesOnlyInBCVariables2() {
+ try{
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(100);
+
+ DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 10).map(new Duplicator<Long>()).name("proper input");
+
+ DataSet<Long> bc_input1 = env.generateSequence(1, 10).name("BC input 1");
+ DataSet<Long> bc_input2 = env.generateSequence(1, 10).name("BC input 1");
+
+ DataSet<Tuple2<Long, Long>> joinInput1 =
+ input.map(new IdentityMapper<Tuple2<Long,Long>>())
+ .withBroadcastSet(bc_input1.map(new IdentityMapper<Long>()), "bc1")
+ .withBroadcastSet(bc_input2, "bc2");
+
+ DataSet<Tuple2<Long, Long>> joinInput2 =
+ input.map(new IdentityMapper<Tuple2<Long,Long>>())
+ .withBroadcastSet(bc_input1, "bc1")
+ .withBroadcastSet(bc_input2, "bc2");
+
+ DataSet<Tuple2<Long, Long>> joinResult = joinInput1
+ .join(joinInput2, JoinHint.REPARTITION_HASH_FIRST).where(0).equalTo(1)
+ .with(new DummyFlatJoinFunction<Tuple2<Long,Long>>());
+
+ input
+ .map(new IdentityMapper<Tuple2<Long,Long>>())
+ .withBroadcastSet(bc_input1, "bc1")
+ .union(joinResult)
+ .print();
+
+ Plan plan = env.createProgramPlan();
+ compileNoStats(plan);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static final class Duplicator<T> implements MapFunction<T, Tuple2<T, T>> {
+
+ @Override
+ public Tuple2<T, T> map(T value) {
+ return new Tuple2<T, T>(value, value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
new file mode 100644
index 0000000..c7ad2da
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/BroadcastVariablePipelinebreakerTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.Plan;
+import org.junit.Test;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.testfunctions.IdentityMapper;
+
+@SuppressWarnings("serial")
+public class BroadcastVariablePipelinebreakerTest extends CompilerTestBase {
+
+ @Test
+ public void testNoBreakerForIndependentVariable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> source1 = env.fromElements("test");
+ DataSet<String> source2 = env.fromElements("test");
+
+ source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name").print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testBreakerForDependentVariable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> source1 = env.fromElements("test");
+
+ source1.map(new IdentityMapper<String>()).map(new IdentityMapper<String>()).withBroadcastSet(source1, "some name").print();
+
+ Plan p = env.createProgramPlan();
+ OptimizedPlan op = compileNoStats(p);
+
+ SinkPlanNode sink = op.getDataSinks().iterator().next();
+ SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
+
+ assertEquals(TempMode.PIPELINE_BREAKER, mapper.getInput().getTempMode());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
new file mode 100644
index 0000000..3e7da6c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CachedMatchStrategyCompilerTest.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.GenericDataSourceBase;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.IterativeDataSet;
+import org.apache.flink.api.common.functions.RichJoinFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.optimizer.dag.TempMode;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+* Tests that validate optimizer choice when using hash joins inside of iterations
+*/
+@SuppressWarnings("serial")
+public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
+
+ /**
+ * This tests whether a HYBRIDHASH_BUILD_SECOND is correctly transformed to a HYBRIDHASH_BUILD_SECOND_CACHED
+ * when inside of an iteration an on the static path
+ */
+ @Test
+ public void testRightSide() {
+ try {
+
+ Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
+
+ // verify correct join strategy
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
+ assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
+ assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ /**
+ * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
+ */
+ @Test
+ public void testRightSideCountercheck() {
+ try {
+
+ Plan plan = getTestPlanRightStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
+
+ // verify correct join strategy
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, innerJoin.getDriverStrategy());
+ assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
+ assertEquals(TempMode.CACHED, innerJoin.getInput2().getTempMode());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ /**
+ * This tests whether a HYBRIDHASH_BUILD_FIRST is correctly transformed to a HYBRIDHASH_BUILD_FIRST_CACHED
+ * when inside of an iteration an on the static path
+ */
+ @Test
+ public void testLeftSide() {
+ try {
+
+ Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
+
+ // verify correct join strategy
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED, innerJoin.getDriverStrategy());
+ assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
+ assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ /**
+ * This test makes sure that only a HYBRIDHASH on the static path is transformed to the cached variant
+ */
+ @Test
+ public void testLeftSideCountercheck() {
+ try {
+
+ Plan plan = getTestPlanLeftStatic(Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
+
+ // verify correct join strategy
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, innerJoin.getDriverStrategy());
+ assertEquals(TempMode.CACHED, innerJoin.getInput1().getTempMode());
+ assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ /**
+ * This test simulates a join of a big left side with a small right side inside of an iteration, where the small side is on a static path.
+ * Currently the best execution plan is a HYBRIDHASH_BUILD_SECOND_CACHED, where the small side is hashed and cached.
+ * This test also makes sure that all relevant plans are correctly enumerated by the optimizer.
+ */
+ @Test
+ public void testCorrectChoosing() {
+ try {
+
+ Plan plan = getTestPlanRightStatic("");
+
+ SourceCollectorVisitor sourceCollector = new SourceCollectorVisitor();
+ plan.accept(sourceCollector);
+
+ for(GenericDataSourceBase<?, ?> s : sourceCollector.getSources()) {
+ if(s.getName().equals("bigFile")) {
+ this.setSourceStatistics(s, 10000000, 1000);
+ }
+ else if(s.getName().equals("smallFile")) {
+ this.setSourceStatistics(s, 100, 100);
+ }
+ }
+
+
+ OptimizedPlan oPlan = compileNoStats(plan);
+
+ OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
+ DualInputPlanNode innerJoin = resolver.getNode("DummyJoiner");
+
+ // verify correct join strategy
+ assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, innerJoin.getDriverStrategy());
+ assertEquals(TempMode.NONE, innerJoin.getInput1().getTempMode());
+ assertEquals(TempMode.NONE, innerJoin.getInput2().getTempMode());
+
+ new JobGraphGenerator().compileJobGraph(oPlan);
+ }
+ catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Test errored: " + e.getMessage());
+ }
+ }
+
+ private Plan getTestPlanRightStatic(String strategy) {
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ DataSet<Tuple3<Long, Long, Long>> bigInput = env.readCsvFile("file://bigFile").types(Long.class, Long.class, Long.class).name("bigFile");
+
+ DataSet<Tuple3<Long, Long, Long>> smallInput = env.readCsvFile("file://smallFile").types(Long.class, Long.class, Long.class).name("smallFile");
+
+ IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
+
+ Configuration joinStrategy = new Configuration();
+ joinStrategy.setString(Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
+
+ if(strategy != "") {
+ joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
+ }
+
+ DataSet<Tuple3<Long, Long, Long>> inner = iteration.join(smallInput).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
+
+ DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
+
+ output.print();
+
+ return env.createProgramPlan();
+
+ }
+
+ private Plan getTestPlanLeftStatic(String strategy) {
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(DEFAULT_PARALLELISM);
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> bigInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L),
+ new Tuple3<Long, Long, Long>(1L, 2L, 3L),new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Big");
+
+ @SuppressWarnings("unchecked")
+ DataSet<Tuple3<Long, Long, Long>> smallInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Small");
+
+ IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
+
+ Configuration joinStrategy = new Configuration();
+ joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
+
+ DataSet<Tuple3<Long, Long, Long>> inner = smallInput.join(iteration).where(0).equalTo(0).with(new DummyJoiner()).name("DummyJoiner").withParameters(joinStrategy);
+
+ DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
+
+ output.print();
+
+ return env.createProgramPlan();
+
+ }
+
+ private static class DummyJoiner extends RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> {
+
+ @Override
+ public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first,
+ Tuple3<Long, Long, Long> second) throws Exception {
+
+ return first;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
new file mode 100644
index 0000000..eba07f1
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/CoGroupSolutionSetFirstTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer;
+
+import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.DeltaIteration;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.Visitor;
+
+@SuppressWarnings("serial")
+public class CoGroupSolutionSetFirstTest extends CompilerTestBase {
+
+ public static class SimpleCGroup extends RichCoGroupFunction<Tuple1<Integer>, Tuple1<Integer>, Tuple1<Integer>> {
+ @Override
+ public void coGroup(Iterable<Tuple1<Integer>> first, Iterable<Tuple1<Integer>> second, Collector<Tuple1<Integer>> out) {}
+ }
+
+ public static class SimpleMap extends RichMapFunction<Tuple1<Integer>, Tuple1<Integer>> {
+ @Override
+ public Tuple1<Integer> map(Tuple1<Integer> value) throws Exception {
+ return null;
+ }
+ }
+
+ @Test
+ public void testCoGroupSolutionSet() {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple1<Integer>> raw = env.readCsvFile(IN_FILE).types(Integer.class);
+
+ DeltaIteration<Tuple1<Integer>, Tuple1<Integer>> iteration = raw.iterateDelta(raw, 1000, 0);
+
+ DataSet<Tuple1<Integer>> test = iteration.getWorkset().map(new SimpleMap());
+ DataSet<Tuple1<Integer>> delta = iteration.getSolutionSet().coGroup(test).where(0).equalTo(0).with(new SimpleCGroup());
+ DataSet<Tuple1<Integer>> feedback = iteration.getWorkset().map(new SimpleMap());
+ DataSet<Tuple1<Integer>> result = iteration.closeWith(delta, feedback);
+
+ result.print();
+
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = null;
+ try {
+ oPlan = compileNoStats(plan);
+ } catch(CompilerException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ oPlan.accept(new Visitor<PlanNode>() {
+ @Override
+ public boolean preVisit(PlanNode visitable) {
+ if (visitable instanceof WorksetIterationPlanNode) {
+ PlanNode deltaNode = ((WorksetIterationPlanNode) visitable).getSolutionSetDeltaPlanNode();
+
+ //get the CoGroup
+ DualInputPlanNode dpn = (DualInputPlanNode) deltaNode.getInputs().iterator().next().getSource();
+ Channel in1 = dpn.getInput1();
+ Channel in2 = dpn.getInput2();
+
+ Assert.assertTrue(in1.getLocalProperties().getOrdering() == null);
+ Assert.assertTrue(in2.getLocalProperties().getOrdering() != null);
+ Assert.assertTrue(in2.getLocalProperties().getOrdering().getInvolvedIndexes().contains(0));
+ Assert.assertTrue(in1.getShipStrategy() == ShipStrategyType.FORWARD);
+ Assert.assertTrue(in2.getShipStrategy() == ShipStrategyType.PARTITION_HASH);
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void postVisit(PlanNode visitable) {}
+ });
+ }
+}
[40/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
[optimizer] Rename optimizer project to "flink-optimizer" (previously flink-compiler)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/633b0d6a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/633b0d6a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/633b0d6a
Branch: refs/heads/master
Commit: 633b0d6a9b25fd324f1aa007bbce246be304c099
Parents: 5757850
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Mar 17 21:44:28 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 20 10:21:14 2015 +0100
----------------------------------------------------------------------
flink-clients/pom.xml | 2 +-
flink-compiler/pom.xml | 63 -
.../flink/optimizer/CompilerException.java | 64 -
.../optimizer/CompilerPostPassException.java | 62 -
.../apache/flink/optimizer/DataStatistics.java | 69 -
.../org/apache/flink/optimizer/Optimizer.java | 571 -------
.../flink/optimizer/costs/CostEstimator.java | 231 ---
.../org/apache/flink/optimizer/costs/Costs.java | 492 ------
.../optimizer/costs/DefaultCostEstimator.java | 265 ---
.../dag/AbstractPartialSolutionNode.java | 104 --
.../flink/optimizer/dag/BinaryUnionNode.java | 308 ----
.../flink/optimizer/dag/BulkIterationNode.java | 390 -----
.../optimizer/dag/BulkPartialSolutionNode.java | 103 --
.../apache/flink/optimizer/dag/CoGroupNode.java | 106 --
.../flink/optimizer/dag/CollectorMapNode.java | 62 -
.../apache/flink/optimizer/dag/CrossNode.java | 138 --
.../flink/optimizer/dag/DagConnection.java | 290 ----
.../flink/optimizer/dag/DataSinkNode.java | 266 ---
.../flink/optimizer/dag/DataSourceNode.java | 306 ----
.../flink/optimizer/dag/EstimateProvider.java | 47 -
.../apache/flink/optimizer/dag/FilterNode.java | 72 -
.../apache/flink/optimizer/dag/FlatMapNode.java | 65 -
.../flink/optimizer/dag/GroupCombineNode.java | 100 --
.../flink/optimizer/dag/GroupReduceNode.java | 158 --
.../dag/InterestingPropertiesClearer.java | 39 -
.../flink/optimizer/dag/IterationNode.java | 30 -
.../apache/flink/optimizer/dag/JoinNode.java | 187 ---
.../org/apache/flink/optimizer/dag/MapNode.java | 66 -
.../flink/optimizer/dag/MapPartitionNode.java | 67 -
.../apache/flink/optimizer/dag/MatchNode.java | 167 --
.../apache/flink/optimizer/dag/NoOpNode.java | 44 -
.../flink/optimizer/dag/OptimizerNode.java | 1172 -------------
.../flink/optimizer/dag/PartitionNode.java | 148 --
.../flink/optimizer/dag/PlanCacheCleaner.java | 39 -
.../apache/flink/optimizer/dag/ReduceNode.java | 98 --
.../flink/optimizer/dag/SingleInputNode.java | 518 ------
.../apache/flink/optimizer/dag/SinkJoiner.java | 112 --
.../flink/optimizer/dag/SolutionSetNode.java | 99 --
.../flink/optimizer/dag/SortPartitionNode.java | 127 --
.../apache/flink/optimizer/dag/TempMode.java | 83 -
.../flink/optimizer/dag/TwoInputNode.java | 747 ---------
.../flink/optimizer/dag/UnaryOperatorNode.java | 69 -
.../optimizer/dag/WorksetIterationNode.java | 589 -------
.../apache/flink/optimizer/dag/WorksetNode.java | 104 --
.../dataproperties/GlobalProperties.java | 500 ------
.../dataproperties/InterestingProperties.java | 179 --
.../dataproperties/LocalProperties.java | 307 ----
.../dataproperties/PartitioningProperty.java | 112 --
.../RequestedGlobalProperties.java | 486 ------
.../RequestedLocalProperties.java | 265 ---
.../optimizer/deadlockdetect/DeadlockEdge.java | 38 -
.../optimizer/deadlockdetect/DeadlockGraph.java | 133 --
.../deadlockdetect/DeadlockPreventer.java | 211 ---
.../deadlockdetect/DeadlockVertex.java | 98 --
.../operators/AbstractJoinDescriptor.java | 185 --
.../operators/AbstractOperatorDescriptor.java | 35 -
.../operators/AllGroupCombineProperties.java | 73 -
.../operators/AllGroupReduceProperties.java | 73 -
.../AllGroupWithPartialPreGroupProperties.java | 100 --
.../operators/AllReduceProperties.java | 94 --
.../operators/BinaryUnionOpDescriptor.java | 97 --
.../operators/CartesianProductDescriptor.java | 110 --
.../optimizer/operators/CoGroupDescriptor.java | 239 ---
.../CoGroupWithSolutionSetFirstDescriptor.java | 77 -
.../CoGroupWithSolutionSetSecondDescriptor.java | 57 -
.../operators/CollectorMapDescriptor.java | 75 -
.../CrossBlockOuterFirstDescriptor.java | 44 -
.../CrossBlockOuterSecondDescriptor.java | 44 -
.../CrossStreamOuterFirstDescriptor.java | 51 -
.../CrossStreamOuterSecondDescriptor.java | 51 -
.../optimizer/operators/FilterDescriptor.java | 68 -
.../optimizer/operators/FlatMapDescriptor.java | 75 -
.../operators/GroupCombineProperties.java | 117 --
.../operators/GroupReduceProperties.java | 129 --
.../GroupReduceWithCombineProperties.java | 169 --
.../operators/HashJoinBuildFirstProperties.java | 89 -
.../HashJoinBuildSecondProperties.java | 88 -
.../optimizer/operators/MapDescriptor.java | 68 -
.../operators/MapPartitionDescriptor.java | 68 -
.../optimizer/operators/NoOpDescriptor.java | 70 -
.../operators/OperatorDescriptorDual.java | 206 ---
.../operators/OperatorDescriptorSingle.java | 106 --
.../operators/PartialGroupProperties.java | 91 -
.../optimizer/operators/ReduceProperties.java | 126 --
.../operators/SolutionSetDeltaOperator.java | 75 -
.../operators/SortMergeJoinDescriptor.java | 110 --
.../operators/UtilSinkJoinOpDescriptor.java | 91 -
.../optimizer/plan/BinaryUnionPlanNode.java | 60 -
.../optimizer/plan/BulkIterationPlanNode.java | 168 --
.../plan/BulkPartialSolutionPlanNode.java | 127 --
.../apache/flink/optimizer/plan/Channel.java | 538 ------
.../flink/optimizer/plan/DualInputPlanNode.java | 246 ---
.../apache/flink/optimizer/plan/FlinkPlan.java | 28 -
.../flink/optimizer/plan/IterationPlanNode.java | 32 -
.../flink/optimizer/plan/NAryUnionPlanNode.java | 106 --
.../flink/optimizer/plan/NamedChannel.java | 46 -
.../flink/optimizer/plan/OptimizedPlan.java | 130 --
.../apache/flink/optimizer/plan/PlanNode.java | 573 -------
.../optimizer/plan/SingleInputPlanNode.java | 271 ---
.../optimizer/plan/SinkJoinerPlanNode.java | 73 -
.../flink/optimizer/plan/SinkPlanNode.java | 50 -
.../optimizer/plan/SolutionSetPlanNode.java | 124 --
.../flink/optimizer/plan/SourcePlanNode.java | 113 --
.../flink/optimizer/plan/StreamingPlan.java | 38 -
.../plan/WorksetIterationPlanNode.java | 259 ---
.../flink/optimizer/plan/WorksetPlanNode.java | 131 --
.../optimizer/plandump/DumpableConnection.java | 32 -
.../flink/optimizer/plandump/DumpableNode.java | 41 -
.../plandump/PlanJSONDumpGenerator.java | 657 --------
.../plantranslate/JobGraphGenerator.java | 1578 ------------------
.../optimizer/postpass/AbstractSchema.java | 39 -
.../ConflictingFieldTypeInfoException.java | 50 -
.../postpass/GenericFlatTypePostPass.java | 579 -------
.../optimizer/postpass/JavaApiPostPass.java | 327 ----
.../postpass/MissingFieldTypeInfoException.java | 34 -
.../optimizer/postpass/OptimizerPostPass.java | 37 -
.../flink/optimizer/postpass/PostPassUtils.java | 47 -
.../optimizer/postpass/RecordModelPostPass.java | 174 --
.../optimizer/postpass/SparseKeySchema.java | 86 -
.../traversals/BinaryUnionReplacer.java | 125 --
.../optimizer/traversals/BranchesVisitor.java | 46 -
.../traversals/GraphCreatingVisitor.java | 392 -----
.../traversals/IdAndEstimatesVisitor.java | 68 -
.../traversals/InterestingPropertyVisitor.java | 60 -
.../optimizer/traversals/PlanFinalizer.java | 229 ---
.../traversals/StaticDynamicPathIdentifier.java | 58 -
.../traversals/StepFunctionValidator.java | 53 -
.../optimizer/traversals/package-info.java | 27 -
.../flink/optimizer/util/NoOpBinaryUdfOp.java | 51 -
.../flink/optimizer/util/NoOpUnaryUdfOp.java | 62 -
.../org/apache/flink/optimizer/util/Utils.java | 81 -
.../optimizer/AdditionalOperatorsTest.java | 110 --
.../optimizer/BranchingPlansCompilerTest.java | 1039 ------------
.../BroadcastVariablePipelinebreakerTest.java | 83 -
.../CachedMatchStrategyCompilerTest.java | 268 ---
.../optimizer/CoGroupSolutionSetFirstTest.java | 101 --
.../flink/optimizer/CompilerTestBase.java | 229 ---
.../apache/flink/optimizer/DOPChangeTest.java | 347 ----
.../flink/optimizer/DisjointDataFlowsTest.java | 51 -
.../optimizer/DistinctCompilationTest.java | 206 ---
.../optimizer/FeedbackPropertiesMatchTest.java | 1436 ----------------
.../apache/flink/optimizer/GroupOrderTest.java | 167 --
.../optimizer/HardPlansCompilationTest.java | 80 -
.../flink/optimizer/IterationsCompilerTest.java | 409 -----
.../flink/optimizer/NestedIterationsTest.java | 181 --
.../flink/optimizer/PartitionPushdownTest.java | 104 --
.../optimizer/PartitioningReusageTest.java | 845 ----------
.../flink/optimizer/PipelineBreakerTest.java | 241 ---
.../flink/optimizer/PropertyDataSourceTest.java | 897 ----------
.../apache/flink/optimizer/ReduceAllTest.java | 61 -
.../optimizer/ReplicatingDataSourceTest.java | 495 ------
.../SemanticPropertiesAPIToPlanTest.java | 173 --
.../flink/optimizer/SortPartialReuseTest.java | 130 --
.../UnionBetweenDynamicAndStaticPathTest.java | 143 --
.../optimizer/UnionPropertyPropagationTest.java | 186 ---
.../flink/optimizer/UnionReplacementTest.java | 55 -
.../WorksetIterationCornerCasesTest.java | 77 -
.../WorksetIterationsRecordApiCompilerTest.java | 247 ---
.../costs/DefaultCostEstimatorTest.java | 428 -----
...naryCustomPartitioningCompatibilityTest.java | 130 --
.../CoGroupCustomPartitioningTest.java | 312 ----
...ustomPartitioningGlobalOptimizationTest.java | 95 --
.../custompartition/CustomPartitioningTest.java | 287 ----
.../GroupingKeySelectorTranslationTest.java | 234 ---
.../GroupingPojoTranslationTest.java | 257 ---
.../GroupingTupleTranslationTest.java | 270 ---
.../JoinCustomPartitioningTest.java | 309 ----
.../DataExchangeModeClosedBranchingTest.java | 257 ---
.../DataExchangeModeForwardTest.java | 139 --
.../DataExchangeModeOpenBranchingTest.java | 182 --
.../dataexchange/PipelineBreakingTest.java | 322 ----
.../GlobalPropertiesFilteringTest.java | 428 -----
.../GlobalPropertiesMatchingTest.java | 294 ----
.../GlobalPropertiesPushdownTest.java | 107 --
.../LocalPropertiesFilteringTest.java | 373 -----
.../dataproperties/MockDistribution.java | 50 -
.../dataproperties/MockPartitioner.java | 32 -
.../RequestedGlobalPropertiesFilteringTest.java | 430 -----
.../RequestedLocalPropertiesFilteringTest.java | 246 ---
.../java/DeltaIterationDependenciesTest.java | 76 -
.../java/DistinctAndGroupingOptimizerTest.java | 112 --
.../java/GroupReduceCompilationTest.java | 368 ----
.../optimizer/java/IterationCompilerTest.java | 189 ---
.../optimizer/java/JoinTranslationTest.java | 168 --
.../flink/optimizer/java/OpenIterationTest.java | 183 --
.../optimizer/java/PartitionOperatorTest.java | 70 -
.../optimizer/java/ReduceCompilationTest.java | 261 ---
.../WorksetIterationsJavaApiCompilerTest.java | 302 ----
...oGroupGlobalPropertiesCompatibilityTest.java | 161 --
.../CoGroupOnConflictingPartitioningsTest.java | 67 -
.../JoinGlobalPropertiesCompatibilityTest.java | 161 --
.../JoinOnConflictingPartitioningsTest.java | 65 -
.../flink/optimizer/plan/ChannelTest.java | 90 -
.../plandump/NumberFormattingTest.java | 53 -
.../testfunctions/DummyCoGroupFunction.java | 31 -
.../testfunctions/DummyFlatJoinFunction.java | 33 -
.../optimizer/testfunctions/DummyReducer.java | 31 -
.../testfunctions/IdentityFlatMapper.java | 30 -
.../testfunctions/IdentityGroupReducer.java | 38 -
.../testfunctions/IdentityKeyExtractor.java | 31 -
.../optimizer/testfunctions/IdentityMapper.java | 32 -
.../IdentityPartitionerMapper.java | 34 -
.../testfunctions/SelectOneReducer.java | 31 -
.../testfunctions/Top1GroupReducer.java | 35 -
.../flink/optimizer/util/DummyCoGroupStub.java | 42 -
.../flink/optimizer/util/DummyCrossStub.java | 32 -
.../flink/optimizer/util/DummyInputFormat.java | 42 -
.../flink/optimizer/util/DummyMatchStub.java | 37 -
.../util/DummyNonPreservingMatchStub.java | 35 -
.../flink/optimizer/util/DummyOutputFormat.java | 34 -
.../flink/optimizer/util/IdentityMap.java | 37 -
.../flink/optimizer/util/IdentityReduce.java | 40 -
.../src/test/resources/log4j-test.properties | 19 -
.../src/test/resources/log4j.properties | 27 -
.../src/test/resources/logback-test.xml | 29 -
flink-dist/pom.xml | 2 +-
flink-optimizer/pom.xml | 63 +
.../flink/optimizer/CompilerException.java | 64 +
.../optimizer/CompilerPostPassException.java | 62 +
.../apache/flink/optimizer/DataStatistics.java | 69 +
.../org/apache/flink/optimizer/Optimizer.java | 571 +++++++
.../flink/optimizer/costs/CostEstimator.java | 231 +++
.../org/apache/flink/optimizer/costs/Costs.java | 492 ++++++
.../optimizer/costs/DefaultCostEstimator.java | 265 +++
.../dag/AbstractPartialSolutionNode.java | 104 ++
.../flink/optimizer/dag/BinaryUnionNode.java | 308 ++++
.../flink/optimizer/dag/BulkIterationNode.java | 390 +++++
.../optimizer/dag/BulkPartialSolutionNode.java | 103 ++
.../apache/flink/optimizer/dag/CoGroupNode.java | 106 ++
.../flink/optimizer/dag/CollectorMapNode.java | 62 +
.../apache/flink/optimizer/dag/CrossNode.java | 138 ++
.../flink/optimizer/dag/DagConnection.java | 290 ++++
.../flink/optimizer/dag/DataSinkNode.java | 266 +++
.../flink/optimizer/dag/DataSourceNode.java | 306 ++++
.../flink/optimizer/dag/EstimateProvider.java | 47 +
.../apache/flink/optimizer/dag/FilterNode.java | 72 +
.../apache/flink/optimizer/dag/FlatMapNode.java | 65 +
.../flink/optimizer/dag/GroupCombineNode.java | 100 ++
.../flink/optimizer/dag/GroupReduceNode.java | 158 ++
.../dag/InterestingPropertiesClearer.java | 39 +
.../flink/optimizer/dag/IterationNode.java | 30 +
.../apache/flink/optimizer/dag/JoinNode.java | 187 +++
.../org/apache/flink/optimizer/dag/MapNode.java | 66 +
.../flink/optimizer/dag/MapPartitionNode.java | 67 +
.../apache/flink/optimizer/dag/MatchNode.java | 167 ++
.../apache/flink/optimizer/dag/NoOpNode.java | 44 +
.../flink/optimizer/dag/OptimizerNode.java | 1172 +++++++++++++
.../flink/optimizer/dag/PartitionNode.java | 148 ++
.../flink/optimizer/dag/PlanCacheCleaner.java | 39 +
.../apache/flink/optimizer/dag/ReduceNode.java | 98 ++
.../flink/optimizer/dag/SingleInputNode.java | 518 ++++++
.../apache/flink/optimizer/dag/SinkJoiner.java | 112 ++
.../flink/optimizer/dag/SolutionSetNode.java | 99 ++
.../flink/optimizer/dag/SortPartitionNode.java | 127 ++
.../apache/flink/optimizer/dag/TempMode.java | 83 +
.../flink/optimizer/dag/TwoInputNode.java | 747 +++++++++
.../flink/optimizer/dag/UnaryOperatorNode.java | 69 +
.../optimizer/dag/WorksetIterationNode.java | 589 +++++++
.../apache/flink/optimizer/dag/WorksetNode.java | 104 ++
.../dataproperties/GlobalProperties.java | 500 ++++++
.../dataproperties/InterestingProperties.java | 179 ++
.../dataproperties/LocalProperties.java | 307 ++++
.../dataproperties/PartitioningProperty.java | 112 ++
.../RequestedGlobalProperties.java | 486 ++++++
.../RequestedLocalProperties.java | 265 +++
.../optimizer/deadlockdetect/DeadlockEdge.java | 38 +
.../optimizer/deadlockdetect/DeadlockGraph.java | 133 ++
.../deadlockdetect/DeadlockPreventer.java | 211 +++
.../deadlockdetect/DeadlockVertex.java | 98 ++
.../operators/AbstractJoinDescriptor.java | 185 ++
.../operators/AbstractOperatorDescriptor.java | 35 +
.../operators/AllGroupCombineProperties.java | 73 +
.../operators/AllGroupReduceProperties.java | 73 +
.../AllGroupWithPartialPreGroupProperties.java | 100 ++
.../operators/AllReduceProperties.java | 94 ++
.../operators/BinaryUnionOpDescriptor.java | 97 ++
.../operators/CartesianProductDescriptor.java | 110 ++
.../optimizer/operators/CoGroupDescriptor.java | 239 +++
.../CoGroupWithSolutionSetFirstDescriptor.java | 77 +
.../CoGroupWithSolutionSetSecondDescriptor.java | 57 +
.../operators/CollectorMapDescriptor.java | 75 +
.../CrossBlockOuterFirstDescriptor.java | 44 +
.../CrossBlockOuterSecondDescriptor.java | 44 +
.../CrossStreamOuterFirstDescriptor.java | 51 +
.../CrossStreamOuterSecondDescriptor.java | 51 +
.../optimizer/operators/FilterDescriptor.java | 68 +
.../optimizer/operators/FlatMapDescriptor.java | 75 +
.../operators/GroupCombineProperties.java | 117 ++
.../operators/GroupReduceProperties.java | 129 ++
.../GroupReduceWithCombineProperties.java | 169 ++
.../operators/HashJoinBuildFirstProperties.java | 89 +
.../HashJoinBuildSecondProperties.java | 88 +
.../optimizer/operators/MapDescriptor.java | 68 +
.../operators/MapPartitionDescriptor.java | 68 +
.../optimizer/operators/NoOpDescriptor.java | 70 +
.../operators/OperatorDescriptorDual.java | 206 +++
.../operators/OperatorDescriptorSingle.java | 106 ++
.../operators/PartialGroupProperties.java | 91 +
.../optimizer/operators/ReduceProperties.java | 126 ++
.../operators/SolutionSetDeltaOperator.java | 75 +
.../operators/SortMergeJoinDescriptor.java | 110 ++
.../operators/UtilSinkJoinOpDescriptor.java | 91 +
.../optimizer/plan/BinaryUnionPlanNode.java | 60 +
.../optimizer/plan/BulkIterationPlanNode.java | 168 ++
.../plan/BulkPartialSolutionPlanNode.java | 127 ++
.../apache/flink/optimizer/plan/Channel.java | 538 ++++++
.../flink/optimizer/plan/DualInputPlanNode.java | 246 +++
.../apache/flink/optimizer/plan/FlinkPlan.java | 28 +
.../flink/optimizer/plan/IterationPlanNode.java | 32 +
.../flink/optimizer/plan/NAryUnionPlanNode.java | 106 ++
.../flink/optimizer/plan/NamedChannel.java | 46 +
.../flink/optimizer/plan/OptimizedPlan.java | 130 ++
.../apache/flink/optimizer/plan/PlanNode.java | 573 +++++++
.../optimizer/plan/SingleInputPlanNode.java | 271 +++
.../optimizer/plan/SinkJoinerPlanNode.java | 73 +
.../flink/optimizer/plan/SinkPlanNode.java | 50 +
.../optimizer/plan/SolutionSetPlanNode.java | 124 ++
.../flink/optimizer/plan/SourcePlanNode.java | 113 ++
.../flink/optimizer/plan/StreamingPlan.java | 38 +
.../plan/WorksetIterationPlanNode.java | 259 +++
.../flink/optimizer/plan/WorksetPlanNode.java | 131 ++
.../optimizer/plandump/DumpableConnection.java | 32 +
.../flink/optimizer/plandump/DumpableNode.java | 41 +
.../plandump/PlanJSONDumpGenerator.java | 657 ++++++++
.../plantranslate/JobGraphGenerator.java | 1578 ++++++++++++++++++
.../optimizer/postpass/AbstractSchema.java | 39 +
.../ConflictingFieldTypeInfoException.java | 50 +
.../postpass/GenericFlatTypePostPass.java | 579 +++++++
.../optimizer/postpass/JavaApiPostPass.java | 327 ++++
.../postpass/MissingFieldTypeInfoException.java | 34 +
.../optimizer/postpass/OptimizerPostPass.java | 37 +
.../flink/optimizer/postpass/PostPassUtils.java | 47 +
.../optimizer/postpass/RecordModelPostPass.java | 174 ++
.../optimizer/postpass/SparseKeySchema.java | 86 +
.../traversals/BinaryUnionReplacer.java | 125 ++
.../optimizer/traversals/BranchesVisitor.java | 46 +
.../traversals/GraphCreatingVisitor.java | 392 +++++
.../traversals/IdAndEstimatesVisitor.java | 68 +
.../traversals/InterestingPropertyVisitor.java | 60 +
.../optimizer/traversals/PlanFinalizer.java | 229 +++
.../traversals/StaticDynamicPathIdentifier.java | 58 +
.../traversals/StepFunctionValidator.java | 53 +
.../optimizer/traversals/package-info.java | 27 +
.../flink/optimizer/util/NoOpBinaryUdfOp.java | 51 +
.../flink/optimizer/util/NoOpUnaryUdfOp.java | 62 +
.../org/apache/flink/optimizer/util/Utils.java | 81 +
.../optimizer/AdditionalOperatorsTest.java | 110 ++
.../optimizer/BranchingPlansCompilerTest.java | 1039 ++++++++++++
.../BroadcastVariablePipelinebreakerTest.java | 83 +
.../CachedMatchStrategyCompilerTest.java | 268 +++
.../optimizer/CoGroupSolutionSetFirstTest.java | 101 ++
.../flink/optimizer/CompilerTestBase.java | 229 +++
.../apache/flink/optimizer/DOPChangeTest.java | 347 ++++
.../flink/optimizer/DisjointDataFlowsTest.java | 51 +
.../optimizer/DistinctCompilationTest.java | 206 +++
.../optimizer/FeedbackPropertiesMatchTest.java | 1436 ++++++++++++++++
.../apache/flink/optimizer/GroupOrderTest.java | 167 ++
.../optimizer/HardPlansCompilationTest.java | 80 +
.../flink/optimizer/IterationsCompilerTest.java | 409 +++++
.../flink/optimizer/NestedIterationsTest.java | 181 ++
.../flink/optimizer/PartitionPushdownTest.java | 104 ++
.../optimizer/PartitioningReusageTest.java | 845 ++++++++++
.../flink/optimizer/PipelineBreakerTest.java | 241 +++
.../flink/optimizer/PropertyDataSourceTest.java | 897 ++++++++++
.../apache/flink/optimizer/ReduceAllTest.java | 61 +
.../optimizer/ReplicatingDataSourceTest.java | 495 ++++++
.../SemanticPropertiesAPIToPlanTest.java | 173 ++
.../flink/optimizer/SortPartialReuseTest.java | 130 ++
.../UnionBetweenDynamicAndStaticPathTest.java | 143 ++
.../optimizer/UnionPropertyPropagationTest.java | 186 +++
.../flink/optimizer/UnionReplacementTest.java | 55 +
.../WorksetIterationCornerCasesTest.java | 77 +
.../WorksetIterationsRecordApiCompilerTest.java | 247 +++
.../costs/DefaultCostEstimatorTest.java | 428 +++++
...naryCustomPartitioningCompatibilityTest.java | 130 ++
.../CoGroupCustomPartitioningTest.java | 312 ++++
...ustomPartitioningGlobalOptimizationTest.java | 95 ++
.../custompartition/CustomPartitioningTest.java | 287 ++++
.../GroupingKeySelectorTranslationTest.java | 234 +++
.../GroupingPojoTranslationTest.java | 257 +++
.../GroupingTupleTranslationTest.java | 270 +++
.../JoinCustomPartitioningTest.java | 309 ++++
.../DataExchangeModeClosedBranchingTest.java | 257 +++
.../DataExchangeModeForwardTest.java | 139 ++
.../DataExchangeModeOpenBranchingTest.java | 182 ++
.../dataexchange/PipelineBreakingTest.java | 322 ++++
.../GlobalPropertiesFilteringTest.java | 428 +++++
.../GlobalPropertiesMatchingTest.java | 294 ++++
.../GlobalPropertiesPushdownTest.java | 107 ++
.../LocalPropertiesFilteringTest.java | 373 +++++
.../dataproperties/MockDistribution.java | 50 +
.../dataproperties/MockPartitioner.java | 32 +
.../RequestedGlobalPropertiesFilteringTest.java | 430 +++++
.../RequestedLocalPropertiesFilteringTest.java | 246 +++
.../java/DeltaIterationDependenciesTest.java | 76 +
.../java/DistinctAndGroupingOptimizerTest.java | 112 ++
.../java/GroupReduceCompilationTest.java | 368 ++++
.../optimizer/java/IterationCompilerTest.java | 189 +++
.../optimizer/java/JoinTranslationTest.java | 168 ++
.../flink/optimizer/java/OpenIterationTest.java | 183 ++
.../optimizer/java/PartitionOperatorTest.java | 70 +
.../optimizer/java/ReduceCompilationTest.java | 261 +++
.../WorksetIterationsJavaApiCompilerTest.java | 302 ++++
...oGroupGlobalPropertiesCompatibilityTest.java | 161 ++
.../CoGroupOnConflictingPartitioningsTest.java | 67 +
.../JoinGlobalPropertiesCompatibilityTest.java | 161 ++
.../JoinOnConflictingPartitioningsTest.java | 65 +
.../flink/optimizer/plan/ChannelTest.java | 90 +
.../plandump/NumberFormattingTest.java | 53 +
.../testfunctions/DummyCoGroupFunction.java | 31 +
.../testfunctions/DummyFlatJoinFunction.java | 33 +
.../optimizer/testfunctions/DummyReducer.java | 31 +
.../testfunctions/IdentityFlatMapper.java | 30 +
.../testfunctions/IdentityGroupReducer.java | 38 +
.../testfunctions/IdentityKeyExtractor.java | 31 +
.../optimizer/testfunctions/IdentityMapper.java | 32 +
.../IdentityPartitionerMapper.java | 34 +
.../testfunctions/SelectOneReducer.java | 31 +
.../testfunctions/Top1GroupReducer.java | 35 +
.../flink/optimizer/util/DummyCoGroupStub.java | 42 +
.../flink/optimizer/util/DummyCrossStub.java | 32 +
.../flink/optimizer/util/DummyInputFormat.java | 42 +
.../flink/optimizer/util/DummyMatchStub.java | 37 +
.../util/DummyNonPreservingMatchStub.java | 35 +
.../flink/optimizer/util/DummyOutputFormat.java | 34 +
.../flink/optimizer/util/IdentityMap.java | 37 +
.../flink/optimizer/util/IdentityReduce.java | 40 +
.../src/test/resources/log4j-test.properties | 19 +
.../src/test/resources/log4j.properties | 27 +
.../src/test/resources/logback-test.xml | 29 +
.../main/resources/archetype-resources/pom.xml | 2 +-
.../main/resources/archetype-resources/pom.xml | 2 +-
flink-scala/pom.xml | 2 +-
flink-staging/flink-streaming/pom.xml | 6 -
flink-test-utils/pom.xml | 2 +-
flink-tests/pom.xml | 2 +-
pom.xml | 2 +-
437 files changed, 38940 insertions(+), 38946 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 95d17d7..6b6d19d 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -50,7 +50,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-compiler</artifactId>
+ <artifactId>flink-optimizer</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/pom.xml
----------------------------------------------------------------------
diff --git a/flink-compiler/pom.xml b/flink-compiler/pom.xml
deleted file mode 100644
index 46d5a9e..0000000
--- a/flink-compiler/pom.xml
+++ /dev/null
@@ -1,63 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-parent</artifactId>
- <version>0.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-compiler</artifactId>
- <name>flink-compiler</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-runtime</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
deleted file mode 100644
index 2f99ddb..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerException.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-/**
- * An exception that is thrown by the Optimizer when encountering an illegal condition.
- */
-public class CompilerException extends RuntimeException {
-
- private static final long serialVersionUID = 3810067304570563755L;
-
- /**
- * Creates a compiler exception with no message and no cause.
- */
- public CompilerException() {}
-
- /**
- * Creates a compiler exception with the given message and no cause.
- *
- * @param message
- * The message for the exception.
- */
- public CompilerException(String message) {
- super(message);
- }
-
- /**
- * Creates a compiler exception with the given cause and no message.
- *
- * @param cause
- * The <tt>Throwable</tt> that caused this exception.
- */
- public CompilerException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Creates a compiler exception with the given message and cause.
- *
- * @param message
- * The message for the exception.
- * @param cause
- * The <tt>Throwable</tt> that caused this exception.
- */
- public CompilerException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
deleted file mode 100644
index 78e47a0..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/CompilerPostPassException.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-/**
- * An exception that is thrown by the Optimizer when encountering
- * a problem during the optimizer post pass. This is a dedicated exception
- * because it is thrown by user-specified optimizer extensions.
- */
-public class CompilerPostPassException extends CompilerException {
-
- private static final long serialVersionUID = -322650826288034623L;
-
- /**
- * Creates a post pass exception with no message and no cause.
- */
- public CompilerPostPassException() {}
-
- /**
- * Creates a post pass exception with the given message and no cause.
- *
- * @param message The message for the exception.
- */
- public CompilerPostPassException(String message) {
- super(message);
- }
-
- /**
- * Creates a post pass exception with the given cause and no message.
- *
- * @param cause The <tt>Throwable</tt> that caused this exception.
- */
- public CompilerPostPassException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Creates a post pass exception with the given message and cause.
- *
- * @param message The message for the exception.
- * @param cause The <tt>Throwable</tt> that caused this exception.
- */
- public CompilerPostPassException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
deleted file mode 100644
index cf6f4ec..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/DataStatistics.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.io.statistics.BaseStatistics;
-
-/**
- * The collection of access methods that can be used to retrieve statistical information about the
- * data processed in a job. Currently this method acts as an entry point only for obtaining cached
- * statistics.
- */
-public class DataStatistics {
-
- private final Map<String, BaseStatistics> baseStatisticsCache;
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Creates a new statistics object, with an empty cache.
- */
- public DataStatistics() {
- this.baseStatisticsCache = new HashMap<String, BaseStatistics>();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the base statistics for the input identified by the given identifier.
- *
- * @param inputIdentifier The identifier for the input.
- * @return The statistics that were cached for this input.
- */
- public BaseStatistics getBaseStatistics(String inputIdentifier) {
- synchronized (this.baseStatisticsCache) {
- return this.baseStatisticsCache.get(inputIdentifier);
- }
- }
-
- /**
- * Caches the given statistics. They are later retrievable under the given identifier.
- *
- * @param statistics The statistics to cache.
- * @param identifier The identifier which may be later used to retrieve the statistics.
- */
- public void cacheBaseStatistics(BaseStatistics statistics, String identifier) {
- synchronized (this.baseStatisticsCache) {
- this.baseStatisticsCache.put(identifier, statistics);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
deleted file mode 100644
index 2101428..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/Optimizer.java
+++ /dev/null
@@ -1,571 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.optimizer.traversals.BinaryUnionReplacer;
-import org.apache.flink.optimizer.traversals.BranchesVisitor;
-import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
-import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
-import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
-import org.apache.flink.optimizer.traversals.PlanFinalizer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.optimizer.costs.CostEstimator;
-import org.apache.flink.optimizer.costs.DefaultCostEstimator;
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.optimizer.deadlockdetect.DeadlockPreventer;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.postpass.OptimizerPostPass;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.util.InstantiationUtil;
-
-/**
- * The optimizer that takes the user specified program plan and creates an optimized plan that contains
- * exact descriptions about how the physical execution will take place. It first translates the user
- * program into an internal optimizer representation and then chooses between different alternatives
- * for shipping strategies and local strategies.
- * <p>
- * The basic principle is taken from optimizer works in systems such as Volcano/Cascades and Selinger/System-R/DB2. The
- * optimizer walks from the sinks down, generating interesting properties, and ascends from the sources generating
- * alternative plans, pruning against the interesting properties.
- * <p>
- * The optimizer also assigns the memory to the individual tasks. This is currently done in a very simple fashion: All
- * sub-tasks that need memory (e.g. reduce or join) are given an equal share of memory.
- */
-public class Optimizer {
-
- // ------------------------------------------------------------------------
- // Constants
- // ------------------------------------------------------------------------
-
- /**
- * Compiler hint key for the input channel's shipping strategy. This String is a key to the operator's stub
- * parameters. The corresponding value tells the compiler which shipping strategy to use for the input channel.
- * If the operator has two input channels, the shipping strategy is applied to both input channels.
- */
- public static final String HINT_SHIP_STRATEGY = "INPUT_SHIP_STRATEGY";
-
- /**
- * Compiler hint key for the <b>first</b> input channel's shipping strategy. This String is a key to
- * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
- * to use for the <b>first</b> input channel. Only applicable to operators with two inputs.
- */
- public static final String HINT_SHIP_STRATEGY_FIRST_INPUT = "INPUT_LEFT_SHIP_STRATEGY";
-
- /**
- * Compiler hint key for the <b>second</b> input channel's shipping strategy. This String is a key to
- * the operator's stub parameters. The corresponding value tells the compiler which shipping strategy
- * to use for the <b>second</b> input channel. Only applicable to operators with two inputs.
- */
- public static final String HINT_SHIP_STRATEGY_SECOND_INPUT = "INPUT_RIGHT_SHIP_STRATEGY";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a <b>Forward</b> strategy on the
- * input channel, i.e. no redistribution of any kind.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_FORWARD = "SHIP_FORWARD";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a random repartition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION= "SHIP_REPARTITION";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a hash-partition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION_HASH = "SHIP_REPARTITION_HASH";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a range-partition strategy.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_REPARTITION_RANGE = "SHIP_REPARTITION_RANGE";
-
- /**
- * Value for the shipping strategy compiler hint that enforces a <b>broadcast</b> strategy on the
- * input channel.
- *
- * @see #HINT_SHIP_STRATEGY
- * @see #HINT_SHIP_STRATEGY_FIRST_INPUT
- * @see #HINT_SHIP_STRATEGY_SECOND_INPUT
- */
- public static final String HINT_SHIP_STRATEGY_BROADCAST = "SHIP_BROADCAST";
-
- /**
- * Compiler hint key for the operator's local strategy. This String is a key to the operator's stub
- * parameters. The corresponding value tells the compiler which local strategy to use to process the
- * data inside one partition.
- * <p>
- * This hint is ignored by operators that do not have a local strategy (such as <i>Map</i>), or by operators that
- * have no choice in their local strategy (such as <i>Cross</i>).
- */
- public static final String HINT_LOCAL_STRATEGY = "LOCAL_STRATEGY";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
- * For example, a <i>Reduce</i> operator will sort the data to group it.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT = "LOCAL_STRATEGY_SORT";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort based</b> local strategy.
- * During sorting a combine method is repeatedly applied to reduce the data volume.
- * For example, a <i>Reduce</i> operator will sort the data to group it.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_COMBINING_SORT = "LOCAL_STRATEGY_COMBINING_SORT";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy on both
- * inputs with subsequent merging of inputs.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE = "LOCAL_STRATEGY_SORT_BOTH_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
- * The first input is sorted, the second input is assumed to be sorted. After sorting both inputs are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE = "LOCAL_STRATEGY_SORT_FIRST_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>sort merge based</b> local strategy.
- * The second input is sorted, the first input is assumed to be sorted. After sorting both inputs are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a sort-merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE = "LOCAL_STRATEGY_SORT_SECOND_MERGE";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>merge based</b> local strategy.
- * Both inputs are assumed to be sorted and are merged.
- * For example, a <i>Match</i> or <i>CoGroup</i> operator will use a merge strategy to find pairs
- * of matching keys.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_MERGE = "LOCAL_STRATEGY_MERGE";
-
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
- * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
- * matching keys. The <b>first</b> input will be used to build the hash table, the second input will be
- * used to probe the table.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST = "LOCAL_STRATEGY_HASH_BUILD_FIRST";
-
- /**
- * Value for the local strategy compiler hint that enforces a <b>hash based</b> local strategy.
- * For example, a <i>Match</i> operator will use a hybrid-hash-join strategy to find pairs of
- * matching keys. The <b>second</b> input will be used to build the hash table, the first input will be
- * used to probe the table.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND = "LOCAL_STRATEGY_HASH_BUILD_SECOND";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
- * Hence, the data of the first input will be is streamed though, while the data of the second input is stored on
- * disk
- * and repeatedly read.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
- * Hence, the data of the second input will be is streamed though, while the data of the first input is stored on
- * disk
- * and repeatedly read.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>first</b> input in the outer-loop of the nested loops.
- * Further more, the first input, being the outer side, will be processed in blocks, and for each block, the second
- * input,
- * being the inner side, will read repeatedly from disk.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST";
-
- /**
- * Value for the local strategy compiler hint that chooses the outer side of the <b>nested-loop</b> local strategy.
- * A <i>Cross</i> operator will process the data of the <b>second</b> input in the outer-loop of the nested loops.
- * Further more, the second input, being the outer side, will be processed in blocks, and for each block, the first
- * input,
- * being the inner side, will read repeatedly from disk.
- *
- * @see #HINT_LOCAL_STRATEGY
- */
- public static final String HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND = "LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND";
-
- /**
- * The log handle that is used by the compiler to log messages.
- */
- public static final Logger LOG = LoggerFactory.getLogger(Optimizer.class);
-
- // ------------------------------------------------------------------------
- // Members
- // ------------------------------------------------------------------------
-
- /**
- * The statistics object used to obtain statistics, such as input sizes,
- * for the cost estimation process.
- */
- private final DataStatistics statistics;
-
- /**
- * The cost estimator used by the compiler.
- */
- private final CostEstimator costEstimator;
-
- /**
- * The default degree of parallelism for jobs compiled by this compiler.
- */
- private int defaultDegreeOfParallelism;
-
-
- // ------------------------------------------------------------------------
- // Constructor & Setup
- // ------------------------------------------------------------------------
-
- /**
- * Creates a new optimizer instance. The optimizer has no access to statistics about the
- * inputs and can hence not determine any properties. It will perform all optimization with
- * unknown sizes and hence use only the heuristic cost functions, which result in the selection
- * of the most robust execution strategies.
- */
- public Optimizer() {
- this(null, new DefaultCostEstimator());
- }
-
- /**
- * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
- * Given those statistics, the optimizer can make better choices for the execution strategies.
- *
- * @param stats
- * The statistics to be used to determine the input properties.
- */
- public Optimizer(DataStatistics stats) {
- this(stats, new DefaultCostEstimator());
- }
-
- /**
- * Creates a new optimizer instance. The optimizer has no access to statistics about the
- * inputs and can hence not determine any properties. It will perform all optimization with
- * unknown sizes and hence use only the heuristic cost functions, which result in the selection
- * of the most robust execution strategies.
- *
- * The optimizer uses the given cost estimator to compute the costs of the individual operations.
- *
- * @param estimator The cost estimator to use to cost the individual operations.
- */
- public Optimizer(CostEstimator estimator) {
- this(null, estimator);
- }
-
- /**
- * Creates a new optimizer instance that uses the statistics object to determine properties about the input.
- * Given those statistics, the optimizer can make better choices for the execution strategies.
- *
- * The optimizer uses the given cost estimator to compute the costs of the individual operations.
- *
- * @param stats
- * The statistics to be used to determine the input properties.
- * @param estimator
- * The <tt>CostEstimator</tt> to use to cost the individual operations.
- */
- public Optimizer(DataStatistics stats, CostEstimator estimator) {
- this.statistics = stats;
- this.costEstimator = estimator;
-
- // determine the default parallelism
- this.defaultDegreeOfParallelism = GlobalConfiguration.getInteger(
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
- ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE);
-
- if (defaultDegreeOfParallelism < 1) {
- LOG.warn("Config value " + defaultDegreeOfParallelism + " for option "
- + ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE + " is invalid. Ignoring and using a value of 1.");
- this.defaultDegreeOfParallelism = 1;
- }
- }
-
- // ------------------------------------------------------------------------
- // Getters / Setters
- // ------------------------------------------------------------------------
-
- public int getDefaultDegreeOfParallelism() {
- return defaultDegreeOfParallelism;
- }
-
- public void setDefaultDegreeOfParallelism(int defaultDegreeOfParallelism) {
- if (defaultDegreeOfParallelism > 0) {
- this.defaultDegreeOfParallelism = defaultDegreeOfParallelism;
- } else {
- throw new IllegalArgumentException("Default parallelism cannot be zero or negative.");
- }
- }
-
- // ------------------------------------------------------------------------
- // Compilation
- // ------------------------------------------------------------------------
-
- /**
- * Translates the given program to an OptimizedPlan, where all nodes have their local strategy assigned
- * and all channels have a shipping strategy assigned.
- *
- * For more details on the optimization phase, see the comments for
- * {@link #compile(org.apache.flink.api.common.Plan, org.apache.flink.optimizer.postpass.OptimizerPostPass)}.
- *
- * @param program The program to be translated.
- * @return The optimized plan.
- *
- * @throws CompilerException
- * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
- * situation during the compilation process.
- */
- public OptimizedPlan compile(Plan program) throws CompilerException {
- final OptimizerPostPass postPasser = getPostPassFromPlan(program);
- return compile(program, postPasser);
- }
-
- /**
- * Translates the given program to an OptimizedPlan. The optimized plan describes for each operator
- * which strategy to use (such as hash join versus sort-merge join), what data exchange method to use
- * (local pipe forward, shuffle, broadcast), what exchange mode to use (pipelined, batch),
- * where to cache intermediate results, etc,
- *
- * The optimization happens in multiple phases:
- * <ol>
- * <li>Create optimizer dag implementation of the program.
- *
- * <tt>OptimizerNode</tt> representations of the PACTs, assign parallelism and compute size estimates.</li>
- * <li>Compute interesting properties and auxiliary structures.</li>
- * <li>Enumerate plan alternatives. This cannot be done in the same step as the interesting property computation (as
- * opposed to the Database approaches), because we support plans that are not trees.</li>
- * </ol>
- *
- * @param program The program to be translated.
- * @param postPasser The function to be used for post passing the optimizer's plan and setting the
- * data type specific serialization routines.
- * @return The optimized plan.
- *
- * @throws CompilerException
- * Thrown, if the plan is invalid or the optimizer encountered an inconsistent
- * situation during the compilation process.
- */
- private OptimizedPlan compile(Plan program, OptimizerPostPass postPasser) throws CompilerException {
- if (program == null || postPasser == null) {
- throw new NullPointerException();
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Beginning compilation of program '" + program.getJobName() + '\'');
- }
-
- final ExecutionMode defaultDataExchangeMode = program.getExecutionConfig().getExecutionMode();
-
- final int defaultParallelism = program.getDefaultParallelism() > 0 ?
- program.getDefaultParallelism() : this.defaultDegreeOfParallelism;
-
- // log the default settings
- LOG.debug("Using a default parallelism of {}", defaultParallelism);
- LOG.debug("Using default data exchange mode {}", defaultDataExchangeMode);
-
- // the first step in the compilation is to create the optimizer plan representation
- // this step does the following:
- // 1) It creates an optimizer plan node for each operator
- // 2) It connects them via channels
- // 3) It looks for hints about local strategies and channel types and
- // sets the types and strategies accordingly
- // 4) It makes estimates about the data volume of the data sources and
- // propagates those estimates through the plan
-
- GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(defaultParallelism, defaultDataExchangeMode);
- program.accept(graphCreator);
-
- // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
- // each until we have only a single root node. This allows to transparently deal with the nodes with
- // multiple outputs
- OptimizerNode rootNode;
- if (graphCreator.getSinks().size() == 1) {
- rootNode = graphCreator.getSinks().get(0);
- }
- else if (graphCreator.getSinks().size() > 1) {
- Iterator<DataSinkNode> iter = graphCreator.getSinks().iterator();
- rootNode = iter.next();
-
- while (iter.hasNext()) {
- rootNode = new SinkJoiner(rootNode, iter.next());
- }
- }
- else {
- throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
- }
-
- // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
- // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
- rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
-
- // We are dealing with operator DAGs, rather than operator trees.
- // That requires us to deviate at some points from the classical DB optimizer algorithms.
- // This step build some auxiliary structures to help track branches and joins in the DAG
- BranchesVisitor branchingVisitor = new BranchesVisitor();
- rootNode.accept(branchingVisitor);
-
- // Propagate the interesting properties top-down through the graph
- InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
- rootNode.accept(propsVisitor);
-
- // perform a sanity check: the root may not have any unclosed branches
- if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
- throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
- "track the re-joining of branches correctly.");
- }
-
- // the final step is now to generate the actual plan alternatives
- List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);
-
- if (bestPlan.size() != 1) {
- throw new CompilerException("Error in compiler: more than one best plan was created!");
- }
-
- // check if the best plan's root is a data sink (single sink plan)
- // if so, directly take it. if it is a sink joiner node, get its contained sinks
- PlanNode bestPlanRoot = bestPlan.get(0);
- List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);
-
- if (bestPlanRoot instanceof SinkPlanNode) {
- bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
- } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {
- ((SinkJoinerPlanNode) bestPlanRoot).getDataSinks(bestPlanSinks);
- }
-
- DeadlockPreventer dp = new DeadlockPreventer();
- dp.resolveDeadlocks(bestPlanSinks);
-
- // finalize the plan
- OptimizedPlan plan = new PlanFinalizer().createFinalPlan(bestPlanSinks, program.getJobName(), program);
-
- plan.accept(new BinaryUnionReplacer());
-
- // post pass the plan. this is the phase where the serialization and comparator code is set
- postPasser.postPass(plan);
-
- return plan;
- }
-
- /**
- * This function performs only the first step to the compilation process - the creation of the optimizer
- * representation of the plan. No estimations or enumerations of alternatives are done here.
- *
- * @param program The plan to generate the optimizer representation for.
- * @return The optimizer representation of the plan, as a collection of all data sinks
- * from the plan can be traversed.
- */
- public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
- GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
- program.accept(graphCreator);
- return graphCreator.getSinks();
- }
-
-
- // ------------------------------------------------------------------------
- // Miscellaneous
- // ------------------------------------------------------------------------
-
- private OptimizerPostPass getPostPassFromPlan(Plan program) {
- final String className = program.getPostPassClassName();
- if (className == null) {
- throw new CompilerException("Optimizer Post Pass class description is null");
- }
- try {
- Class<? extends OptimizerPostPass> clazz = Class.forName(className).asSubclass(OptimizerPostPass.class);
- try {
- return InstantiationUtil.instantiate(clazz, OptimizerPostPass.class);
- } catch (RuntimeException rtex) {
- // unwrap the source exception
- if (rtex.getCause() != null) {
- throw new CompilerException("Cannot instantiate optimizer post pass: " + rtex.getMessage(), rtex.getCause());
- } else {
- throw rtex;
- }
- }
- }
- catch (ClassNotFoundException cnfex) {
- throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex);
- }
- catch (ClassCastException ccex) {
- throw new CompilerException("Class '" + className + "' is not an optimizer post-pass.", ccex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
deleted file mode 100644
index 7880734..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/costs/CostEstimator.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.costs;
-
-import java.util.Iterator;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.EstimateProvider;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.PlanNode;
-
-/**
- * Abstract base class for a cost estimator. Defines cost estimation methods and implements the basic work
- * method that computes the cost of an operator by adding input shipping cost, input local cost, and
- * driver cost.
- */
-public abstract class CostEstimator {
-
- public abstract void addRandomPartitioningCost(EstimateProvider estimates, Costs costs);
-
- public abstract void addHashPartitioningCost(EstimateProvider estimates, Costs costs);
-
- public abstract void addRangePartitionCost(EstimateProvider estimates, Costs costs);
-
- public abstract void addBroadcastCost(EstimateProvider estimates, int replicationFactor, Costs costs);
-
- // ------------------------------------------------------------------------
-
- public abstract void addFileInputCost(long fileSizeInBytes, Costs costs);
-
- public abstract void addLocalSortCost(EstimateProvider estimates, Costs costs);
-
- public abstract void addLocalMergeCost(EstimateProvider estimates1, EstimateProvider estimates2, Costs costs, int costWeight);
-
- public abstract void addHybridHashCosts(EstimateProvider buildSide, EstimateProvider probeSide, Costs costs, int costWeight);
-
- public abstract void addCachedHybridHashCosts(EstimateProvider buildSide, EstimateProvider probeSide, Costs costs, int costWeight);
-
- public abstract void addStreamedNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long bufferSize, Costs costs, int costWeight);
-
- public abstract void addBlockNestedLoopsCosts(EstimateProvider outerSide, EstimateProvider innerSide, long blockSize, Costs costs, int costWeight);
-
- // ------------------------------------------------------------------------
-
- public abstract void addArtificialDamCost(EstimateProvider estimates, long bufferSize, Costs costs);
-
- // ------------------------------------------------------------------------
-
- /**
- * This method computes the cost of an operator. The cost is composed of cost for input shipping,
- * locally processing an input, and running the operator.
- *
- * It requires at least that all inputs are set and have a proper ship strategy set,
- * which is not equal to <tt>NONE</tt>.
- *
- * @param n The node to compute the costs for.
- */
- public void costOperator(PlanNode n) {
- // initialize costs objects with no costs
- final Costs totalCosts = new Costs();
- final long availableMemory = n.getGuaranteedAvailableMemory();
-
- // add the shipping strategy costs
- for (Channel channel : n.getInputs()) {
- final Costs costs = new Costs();
-
- // Plans that apply the same strategies, but at different points
- // are equally expensive. For example, if a partitioning can be
- // pushed below a Map function there is often no difference in plan
- // costs between the pushed down version and the version that partitions
- // after the Mapper. However, in those cases, we want the expensive
- // strategy to appear later in the plan, as data reduction often occurs
- // by large factors, while blowup is rare and typically by smaller fractions.
- // We achieve this by adding a penalty to small penalty to the FORWARD strategy,
- // weighted by the current plan depth (steps to the earliest data source).
- // that way, later FORWARDS are more expensive than earlier forwards.
- // Note that this only applies to the heuristic costs.
-
- switch (channel.getShipStrategy()) {
- case NONE:
- throw new CompilerException(
- "Cannot determine costs: Shipping strategy has not been set for an input.");
- case FORWARD:
-// costs.addHeuristicNetworkCost(channel.getMaxDepth());
- break;
- case PARTITION_RANDOM:
- addRandomPartitioningCost(channel, costs);
- break;
- case PARTITION_HASH:
- case PARTITION_CUSTOM:
- addHashPartitioningCost(channel, costs);
- break;
- case PARTITION_RANGE:
- addRangePartitionCost(channel, costs);
- break;
- case BROADCAST:
- addBroadcastCost(channel, channel.getReplicationFactor(), costs);
- break;
- case PARTITION_FORCED_REBALANCE:
- addRandomPartitioningCost(channel, costs);
- break;
- default:
- throw new CompilerException("Unknown shipping strategy for input: " + channel.getShipStrategy());
- }
-
- switch (channel.getLocalStrategy()) {
- case NONE:
- break;
- case SORT:
- case COMBININGSORT:
- addLocalSortCost(channel, costs);
- break;
- default:
- throw new CompilerException("Unsupported local strategy for input: " + channel.getLocalStrategy());
- }
-
- if (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE) {
- addArtificialDamCost(channel, 0, costs);
- }
-
- // adjust with the cost weight factor
- if (channel.isOnDynamicPath()) {
- costs.multiplyWith(channel.getCostWeight());
- }
-
- totalCosts.addCosts(costs);
- }
-
- Channel firstInput = null;
- Channel secondInput = null;
- Costs driverCosts = new Costs();
- int costWeight = 1;
-
- // adjust with the cost weight factor
- if (n.isOnDynamicPath()) {
- costWeight = n.getCostWeight();
- }
-
- // get the inputs, if we have some
- {
- Iterator<Channel> channels = n.getInputs().iterator();
- if (channels.hasNext()) {
- firstInput = channels.next();
- }
- if (channels.hasNext()) {
- secondInput = channels.next();
- }
- }
-
- // determine the local costs
- switch (n.getDriverStrategy()) {
- case NONE:
- case UNARY_NO_OP:
- case BINARY_NO_OP:
- case COLLECTOR_MAP:
- case MAP:
- case MAP_PARTITION:
- case FLAT_MAP:
-
- case ALL_GROUP_REDUCE:
- case ALL_REDUCE:
- // this operations does not do any actual grouping, since every element is in the same single group
-
- case CO_GROUP:
- case SORTED_GROUP_REDUCE:
- case SORTED_REDUCE:
- // grouping or co-grouping over sorted streams for free
-
- case SORTED_GROUP_COMBINE:
- // partial grouping is always local and main memory resident. we should add a relative cpu cost at some point
-
- // partial grouping is always local and main memory resident. we should add a relative cpu cost at some point
- case ALL_GROUP_COMBINE:
-
- case UNION:
- // pipelined local union is for free
-
- break;
- case MERGE:
- addLocalMergeCost(firstInput, secondInput, driverCosts, costWeight);
- break;
- case HYBRIDHASH_BUILD_FIRST:
- addHybridHashCosts(firstInput, secondInput, driverCosts, costWeight);
- break;
- case HYBRIDHASH_BUILD_SECOND:
- addHybridHashCosts(secondInput, firstInput, driverCosts, costWeight);
- break;
- case HYBRIDHASH_BUILD_FIRST_CACHED:
- addCachedHybridHashCosts(firstInput, secondInput, driverCosts, costWeight);
- break;
- case HYBRIDHASH_BUILD_SECOND_CACHED:
- addCachedHybridHashCosts(secondInput, firstInput, driverCosts, costWeight);
- break;
- case NESTEDLOOP_BLOCKED_OUTER_FIRST:
- addBlockNestedLoopsCosts(firstInput, secondInput, availableMemory, driverCosts, costWeight);
- break;
- case NESTEDLOOP_BLOCKED_OUTER_SECOND:
- addBlockNestedLoopsCosts(secondInput, firstInput, availableMemory, driverCosts, costWeight);
- break;
- case NESTEDLOOP_STREAMED_OUTER_FIRST:
- addStreamedNestedLoopsCosts(firstInput, secondInput, availableMemory, driverCosts, costWeight);
- break;
- case NESTEDLOOP_STREAMED_OUTER_SECOND:
- addStreamedNestedLoopsCosts(secondInput, firstInput, availableMemory, driverCosts, costWeight);
- break;
- default:
- throw new CompilerException("Unknown local strategy: " + n.getDriverStrategy().name());
- }
-
- totalCosts.addCosts(driverCosts);
- n.setCosts(totalCosts);
- }
-}
[35/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
deleted file mode 100644
index 6946641..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-
-/**
- * Interesting properties are propagated from parent operators to child operators. They tell the child
- * what data properties would help the parent in operating in a cheaper fashion. A reduce operator, for
- * example, tells its child that partitioned data would help. If the child is a join operator, it can use
- * that knowledge to favor strategies that leave the data in a partitioned form.
- *
- * More on optimization with interesting properties can be found in the works on
- * the volcano- and cascades optimizer framework.
- */
-public class InterestingProperties implements Cloneable {
-
- private Set<RequestedGlobalProperties> globalProps; // the global properties, i.e. properties across partitions
-
- private Set<RequestedLocalProperties> localProps; // the local properties, i.e. properties within partitions
-
- // ------------------------------------------------------------------------
-
- public InterestingProperties() {
- this.globalProps = new HashSet<RequestedGlobalProperties>();
- this.localProps = new HashSet<RequestedLocalProperties>();
- }
-
- /**
- * Private constructor for cloning purposes.
- *
- * @param globalProps The global properties for this new object.
- * @param localProps The local properties for this new object.
- */
- private InterestingProperties(Set<RequestedGlobalProperties> globalProps, Set<RequestedLocalProperties> localProps) {
- this.globalProps = globalProps;
- this.localProps = localProps;
- }
-
- // ------------------------------------------------------------------------
-
- public void addGlobalProperties(RequestedGlobalProperties props) {
- this.globalProps.add(props);
- }
-
- public void addLocalProperties(RequestedLocalProperties props) {
- this.localProps.add(props);
- }
-
- public void addInterestingProperties(InterestingProperties other) {
- this.globalProps.addAll(other.globalProps);
- this.localProps.addAll(other.localProps);
- }
-
- /**
- * Gets the interesting local properties.
- *
- * @return The interesting local properties.
- */
- public Set<RequestedLocalProperties> getLocalProperties() {
- return this.localProps;
- }
-
- /**
- * Gets the interesting global properties.
- *
- * @return The interesting global properties.
- */
- public Set<RequestedGlobalProperties> getGlobalProperties() {
- return this.globalProps;
- }
-
- public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input) {
- InterestingProperties iProps = new InterestingProperties();
- SemanticProperties props;
- if (node instanceof SingleInputNode || node instanceof TwoInputNode) {
- props = node.getSemanticProperties();
- } else {
- props = new SemanticProperties.EmptySemanticProperties();
- }
-
- for (RequestedGlobalProperties rgp : this.globalProps) {
- RequestedGlobalProperties filtered = rgp.filterBySemanticProperties(props, input);
- if (filtered != null && !filtered.isTrivial()) {
- iProps.addGlobalProperties(filtered);
- }
- }
- for (RequestedLocalProperties rlp : this.localProps) {
- RequestedLocalProperties filtered = rlp.filterBySemanticProperties(props, input);
- if (filtered != null && !filtered.isTrivial()) {
- iProps.addLocalProperties(filtered);
- }
- }
- return iProps;
- }
-
- public void dropTrivials() {
- for (Iterator<RequestedGlobalProperties> iter = this.globalProps.iterator(); iter.hasNext();) {
- RequestedGlobalProperties gp = iter.next();
- if (gp.isTrivial()) {
- iter.remove();
- break;
- }
- }
-
- for (Iterator<RequestedLocalProperties> iter = this.localProps.iterator(); iter.hasNext();) {
- RequestedLocalProperties lp = iter.next();
- if (lp.isTrivial()) {
- iter.remove();
- break;
- }
- }
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((globalProps == null) ? 0 : globalProps.hashCode());
- result = prime * result + ((localProps == null) ? 0 : localProps.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj != null && obj instanceof InterestingProperties) {
- InterestingProperties other = (InterestingProperties) obj;
- return this.globalProps.equals(other.globalProps) &&
- this.localProps.equals(other.localProps);
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "InterestingProperties [globalProps=" + this.globalProps +
- ", localProps=" + this.localProps + " ]";
- }
-
- @Override
- public InterestingProperties clone() {
- HashSet<RequestedGlobalProperties> globalProps = new HashSet<RequestedGlobalProperties>();
- for (RequestedGlobalProperties p : this.globalProps) {
- globalProps.add(p.clone());
- }
- HashSet<RequestedLocalProperties> localProps = new HashSet<RequestedLocalProperties>();
- for (RequestedLocalProperties p : this.localProps) {
- localProps.add(p.clone());
- }
-
- return new InterestingProperties(globalProps, localProps);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
deleted file mode 100644
index e0231aa..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dataproperties;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class represents local properties of the data. A local property is a property that exists
- * within the data of a single partition, such as sort order, or data grouping.
- */
-public class LocalProperties implements Cloneable {
-
- public static final Logger LOG = LoggerFactory.getLogger(GlobalProperties.class);
-
- public static final LocalProperties EMPTY = new LocalProperties();
-
- // --------------------------------------------------------------------------------------------
-
- private Ordering ordering; // order inside a partition, null if not ordered
-
- private FieldList groupedFields; // fields by which the stream is grouped. null if not grouped.
-
- private Set<FieldSet> uniqueFields; // fields whose value combination is unique in the stream
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Default constructor for trivial local properties. No order, no grouping, no uniqueness.
- */
- public LocalProperties() {}
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the key order.
- *
- * @return The key order, or <code>null</code> if nothing is ordered.
- */
- public Ordering getOrdering() {
- return ordering;
- }
-
- /**
- * Gets the grouped fields.
- *
- * @return The grouped fields, or <code>null</code> if nothing is grouped.
- */
- public FieldList getGroupedFields() {
- return this.groupedFields;
- }
-
- /**
- * Gets the fields whose combination is unique within the data set.
- *
- * @return The unique field combination, or <code>null</code> if nothing is unique.
- */
- public Set<FieldSet> getUniqueFields() {
- return this.uniqueFields;
- }
-
- /**
- * Checks whether the given set of fields is unique, as specified in these local properties.
- *
- * @param set The set to check.
- * @return True, if the given column combination is unique, false if not.
- */
- public boolean areFieldsUnique(FieldSet set) {
- return this.uniqueFields != null && this.uniqueFields.contains(set);
- }
-
- /**
- * Adds a combination of fields that are unique in these data properties.
- *
- * @param uniqueFields The fields that are unique in these data properties.
- */
- public LocalProperties addUniqueFields(FieldSet uniqueFields) {
- LocalProperties copy = clone();
-
- if (copy.uniqueFields == null) {
- copy.uniqueFields = new HashSet<FieldSet>();
- }
- copy.uniqueFields.add(uniqueFields);
- return copy;
- }
-
- public LocalProperties clearUniqueFieldSets() {
- if (this.uniqueFields == null || this.uniqueFields.isEmpty()) {
- return this;
- } else {
- LocalProperties copy = new LocalProperties();
- copy.ordering = this.ordering;
- copy.groupedFields = this.groupedFields;
- return copy;
- }
- }
-
- /**
- * Checks, if the properties in this object are trivial, i.e. only standard values.
- */
- public boolean isTrivial() {
- return ordering == null && this.groupedFields == null && this.uniqueFields == null;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Filters these LocalProperties by the fields that are forwarded to the output
- * as described by the SemanticProperties.
- *
- * @param props The semantic properties holding information about forwarded fields.
- * @param input The index of the input.
- * @return The filtered LocalProperties
- */
- public LocalProperties filterBySemanticProperties(SemanticProperties props, int input) {
-
- if (props == null) {
- throw new NullPointerException("SemanticProperties may not be null.");
- }
-
- LocalProperties returnProps = new LocalProperties();
-
- // check if sorting is preserved
- if (this.ordering != null) {
- Ordering newOrdering = new Ordering();
-
- for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
- int sourceField = this.ordering.getInvolvedIndexes().get(i);
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
- if (targetField == null || targetField.size() == 0) {
- if (i == 0) {
- // order fully destroyed
- newOrdering = null;
- break;
- } else {
- // order partially preserved
- break;
- }
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
- }
- }
-
- returnProps.ordering = newOrdering;
- if (newOrdering != null) {
- returnProps.groupedFields = newOrdering.getInvolvedIndexes();
- } else {
- returnProps.groupedFields = null;
- }
- }
- // check if grouping is preserved
- else if (this.groupedFields != null) {
- FieldList newGroupedFields = new FieldList();
-
- for (Integer sourceField : this.groupedFields) {
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
- if (targetField == null || targetField.size() == 0) {
- newGroupedFields = null;
- break;
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newGroupedFields = newGroupedFields.addField(targetField.toArray()[0]);
- }
- }
- returnProps.groupedFields = newGroupedFields;
- }
-
- if (this.uniqueFields != null) {
- Set<FieldSet> newUniqueFields = new HashSet<FieldSet>();
- for (FieldSet fields : this.uniqueFields) {
- FieldSet newFields = new FieldSet();
- for (Integer sourceField : fields) {
- FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
-
- if (targetField == null || targetField.size() == 0) {
- newFields = null;
- break;
- } else {
- // use any field of target fields for now. We should use something like field equivalence sets in the future.
- if(targetField.size() > 1) {
- LOG.warn("Found that a field is forwarded to more than one target field in " +
- "semantic forwarded field information. Will only use the field with the lowest index.");
- }
- newFields = newFields.addField(targetField.toArray()[0]);
- }
- }
- if (newFields != null) {
- newUniqueFields.add(newFields);
- }
- }
-
- if (!newUniqueFields.isEmpty()) {
- returnProps.uniqueFields = newUniqueFields;
- } else {
- returnProps.uniqueFields = null;
- }
- }
-
- return returnProps;
- }
- // --------------------------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (this.ordering == null ? 0 : this.ordering.hashCode());
- result = prime * result + (this.groupedFields == null ? 0 : this.groupedFields.hashCode());
- result = prime * result + (this.uniqueFields == null ? 0 : this.uniqueFields.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof LocalProperties) {
- final LocalProperties other = (LocalProperties) obj;
- return (ordering == other.getOrdering() || (ordering != null && ordering.equals(other.getOrdering()))) &&
- (groupedFields == other.getGroupedFields() || (groupedFields != null && groupedFields.equals(other.getGroupedFields()))) &&
- (uniqueFields == other.getUniqueFields() || (uniqueFields != null && uniqueFields.equals(other.getUniqueFields())));
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "LocalProperties [ordering=" + this.ordering + ", grouped=" + this.groupedFields
- + ", unique=" + this.uniqueFields + "]";
- }
-
- @Override
- public LocalProperties clone() {
- LocalProperties copy = new LocalProperties();
- copy.ordering = this.ordering;
- copy.groupedFields = this.groupedFields;
- copy.uniqueFields = (this.uniqueFields == null ? null : new HashSet<FieldSet>(this.uniqueFields));
- return copy;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static LocalProperties combine(LocalProperties lp1, LocalProperties lp2) {
- if (lp1.ordering != null) {
- return lp1;
- } else if (lp2.ordering != null) {
- return lp2;
- } else if (lp1.groupedFields != null) {
- return lp1;
- } else if (lp2.groupedFields != null) {
- return lp2;
- } else if (lp1.uniqueFields != null && !lp1.uniqueFields.isEmpty()) {
- return lp1;
- } else if (lp2.uniqueFields != null && !lp2.uniqueFields.isEmpty()) {
- return lp2;
- } else {
- return lp1;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static LocalProperties forOrdering(Ordering o) {
- LocalProperties props = new LocalProperties();
- props.ordering = o;
- props.groupedFields = o.getInvolvedIndexes();
- return props;
- }
-
- public static LocalProperties forGrouping(FieldList groupedFields) {
- LocalProperties props = new LocalProperties();
- props.groupedFields = groupedFields;
- return props;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
deleted file mode 100644
index 5e06dd3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-/**
- * An enumeration of the the different types of distributing data across partitions or
- * parallel workers.
- */
-public enum PartitioningProperty {
-
- /**
- * Any possible way of data distribution, including random partitioning and full replication.
- */
- ANY_DISTRIBUTION,
-
- /**
- * A random disjunct (non-replicated) data distribution, where each datum is contained in one partition only.
- * This is for example the result of parallel scans of data in a file system like HDFS,
- * or the result of a round-robin data distribution.
- */
- RANDOM_PARTITIONED,
-
- /**
- * A hash partitioning on a certain key.
- */
- HASH_PARTITIONED,
-
- /**
- * A range partitioning on a certain key.
- */
- RANGE_PARTITIONED,
-
- /**
- * A not further specified partitioning on a key (hash-, or range partitioning, or some other scheme even).
- */
- ANY_PARTITIONING,
-
- /**
- *Full replication of the data to each parallel instance.
- */
- FULL_REPLICATION,
-
- /**
- * A forced even re-balancing. All partitions are guaranteed to have almost the same number of records.
- */
- FORCED_REBALANCED,
-
- /**
- * A custom partitioning, accompanied by a {@link org.apache.flink.api.common.functions.Partitioner}.
- */
- CUSTOM_PARTITIONING;
-
- /**
- * Checks, if this property represents in fact a partitioning. That is,
- * whether this property is not equal to <tt>PartitionProperty.FULL_REPLICATION</tt>.
- *
- * @return True, if this enum constant is unequal to <tt>PartitionProperty.FULL_REPLICATION</tt>,
- * false otherwise.
- */
- public boolean isPartitioned() {
- return this != FULL_REPLICATION && this != FORCED_REBALANCED && this != ANY_DISTRIBUTION;
- }
-
- /**
- * Checks, if this property represents a full replication.
- *
- * @return True, if this enum constant is equal to <tt>PartitionProperty.FULL_REPLICATION</tt>,
- * false otherwise.
- */
- public boolean isReplication() {
- return this == FULL_REPLICATION;
- }
-
- /**
- * Checks if this property presents a partitioning that is not random, but on a partitioning key.
- *
- * @return True, if the data is partitioned on a key.
- */
- public boolean isPartitionedOnKey() {
- return isPartitioned() && this != RANDOM_PARTITIONED;
- }
-
- /**
- * Checks, if this property represents a partitioning that is computable.
- * A computable partitioning can be recreated through an algorithm. If two sets of data are to
- * be co-partitioned, it is crucial, that the partitioning schemes are computable.
- * <p>
- * Examples for computable partitioning schemes are hash- or range-partitioning. An example for a non-computable
- * partitioning is the implicit partitioning that exists though a globally unique key.
- *
- * @return True, if this enum constant is a re-computable partitioning.
- */
- public boolean isComputablyPartitioned() {
- return this == HASH_PARTITIONED || this == RANGE_PARTITIONED || this == CUSTOM_PARTITIONING;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
deleted file mode 100644
index 8c3f6bd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
+++ /dev/null
@@ -1,486 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-/**
- * This class represents the global properties of the data that are requested by an operator.
- * Operators request the global properties they need for correct execution. This list is an example of global
- * properties requested by certain operators:
- * <ul>
- * <li>"groupBy/reduce" will request the data to be partitioned in some way after the key fields.</li>
- * <li>"map" will request the data to be in an arbitrary distribution - it has no prerequisites</li>
- * <li>"join" will request certain properties for each input. This class represents the properties
- * on an input alone. The properties may be partitioning on the key fields, or a combination of
- * replication on one input and anything-but-replication on the other input.</li>
- * </ul>
- */
-public final class RequestedGlobalProperties implements Cloneable {
-
- private PartitioningProperty partitioning; // the type partitioning
-
- private FieldSet partitioningFields; // the fields which are partitioned
-
- private Ordering ordering; // order of the partitioned fields, if it is an ordered (range) range partitioning
-
- private DataDistribution dataDistribution; // optional data distribution, for a range partitioning
-
- private Partitioner<?> customPartitioner; // optional, partitioner for custom partitioning
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Initializes the global properties with no partitioning.
- */
- public RequestedGlobalProperties() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Sets these properties to request a hash partitioning on the given fields.
- *
- * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
- * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
- * then only an exact partitioning on the fields matches this requested partitioning.
- *
- * @param partitionedFields The key fields for the partitioning.
- */
- public void setHashPartitioned(FieldSet partitionedFields) {
- if (partitionedFields == null) {
- throw new NullPointerException();
- }
- this.partitioning = PartitioningProperty.HASH_PARTITIONED;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- }
-
-
- public void setRangePartitioned(Ordering ordering) {
- this.setRangePartitioned(ordering, null);
- }
-
- public void setRangePartitioned(Ordering ordering, DataDistribution dataDistribution) {
- if (ordering == null) {
- throw new NullPointerException();
- }
- this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
- this.ordering = ordering;
- this.partitioningFields = null;
- this.dataDistribution = dataDistribution;
- }
-
- /**
- * Sets these properties to request some partitioning on the given fields. This will allow
- * both hash partitioning and range partitioning to match.
- *
- * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
- * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
- * then only an exact partitioning on the fields matches this requested partitioning.
- *
- * @param partitionedFields The key fields for the partitioning.
- */
- public void setAnyPartitioning(FieldSet partitionedFields) {
- if (partitionedFields == null) {
- throw new NullPointerException();
- }
- this.partitioning = PartitioningProperty.ANY_PARTITIONING;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- }
-
- public void setRandomPartitioning() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setAnyDistribution() {
- this.partitioning = PartitioningProperty.ANY_DISTRIBUTION;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setFullyReplicated() {
- this.partitioning = PartitioningProperty.FULL_REPLICATION;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- public void setForceRebalancing() {
- this.partitioning = PartitioningProperty.FORCED_REBALANCED;
- this.partitioningFields = null;
- this.ordering = null;
- }
-
- /**
- * Sets these properties to request a custom partitioning with the given {@link Partitioner} instance.
- *
- * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
- * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
- * then only an exact partitioning on the fields matches this requested partitioning.
- *
- * @param partitionedFields The key fields for the partitioning.
- */
- public void setCustomPartitioned(FieldSet partitionedFields, Partitioner<?> partitioner) {
- if (partitionedFields == null || partitioner == null) {
- throw new NullPointerException();
- }
-
- this.partitioning = PartitioningProperty.CUSTOM_PARTITIONING;
- this.partitioningFields = partitionedFields;
- this.ordering = null;
- this.customPartitioner = partitioner;
- }
-
- /**
- * Gets the partitioning property.
- *
- * @return The partitioning property.
- */
- public PartitioningProperty getPartitioning() {
- return partitioning;
- }
-
- /**
- * Gets the fields on which the data is partitioned.
- *
- * @return The partitioning fields.
- */
- public FieldSet getPartitionedFields() {
- return this.partitioningFields;
- }
-
- /**
- * Gets the key order.
- *
- * @return The key order.
- */
- public Ordering getOrdering() {
- return this.ordering;
- }
-
- /**
- * Gets the data distribution.
- *
- * @return The data distribution.
- */
- public DataDistribution getDataDistribution() {
- return this.dataDistribution;
- }
-
- /**
- * Gets the custom partitioner associated with these properties.
- *
- * @return The custom partitioner associated with these properties.
- */
- public Partitioner<?> getCustomPartitioner() {
- return customPartitioner;
- }
-
- /**
- * Checks, if the properties in this object are trivial, i.e. only standard values.
- */
- public boolean isTrivial() {
- return this.partitioning == null || this.partitioning == PartitioningProperty.RANDOM_PARTITIONED;
- }
-
- /**
- * This method resets the properties to a state where no properties are given.
- */
- public void reset() {
- this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
- this.ordering = null;
- this.partitioningFields = null;
- this.dataDistribution = null;
- this.customPartitioner = null;
- }
-
- /**
- * Filters these properties by what can be preserved by the given SemanticProperties when propagated down
- * to the given input.
- *
- * @param props The SemanticProperties which define which fields are preserved.
- * @param input The index of the operator's input.
- * @return The filtered RequestedGlobalProperties
- */
- public RequestedGlobalProperties filterBySemanticProperties(SemanticProperties props, int input) {
- // no semantic properties available. All global properties are filtered.
- if (props == null) {
- throw new NullPointerException("SemanticProperties may not be null.");
- }
-
- RequestedGlobalProperties rgProp = new RequestedGlobalProperties();
-
- switch(this.partitioning) {
- case FULL_REPLICATION:
- case FORCED_REBALANCED:
- case CUSTOM_PARTITIONING:
- case RANDOM_PARTITIONED:
- case ANY_DISTRIBUTION:
- // make sure that certain properties are not pushed down
- return null;
- case HASH_PARTITIONED:
- case ANY_PARTITIONING:
- FieldSet newFields;
- if(this.partitioningFields instanceof FieldList) {
- newFields = new FieldList();
- } else {
- newFields = new FieldSet();
- }
-
- for (Integer targetField : this.partitioningFields) {
- int sourceField = props.getForwardingSourceField(input, targetField);
- if (sourceField >= 0) {
- newFields = newFields.addField(sourceField);
- } else {
- // partial partitionings are not preserved to avoid skewed partitioning
- return null;
- }
- }
- rgProp.partitioning = this.partitioning;
- rgProp.partitioningFields = newFields;
- return rgProp;
- case RANGE_PARTITIONED:
- // range partitioning
- Ordering newOrdering = new Ordering();
- for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
- int value = this.ordering.getInvolvedIndexes().get(i);
- int sourceField = props.getForwardingSourceField(input, value);
- if (sourceField >= 0) {
- newOrdering.appendOrdering(sourceField, this.ordering.getType(i), this.ordering.getOrder(i));
- } else {
- return null;
- }
- }
- rgProp.partitioning = this.partitioning;
- rgProp.ordering = newOrdering;
- rgProp.dataDistribution = this.dataDistribution;
- return rgProp;
- default:
- throw new RuntimeException("Unknown partitioning type encountered.");
- }
- }
-
- /**
- * Checks, if this set of interesting properties, is met by the given
- * produced properties.
- *
- * @param props The properties for which to check whether they meet these properties.
- * @return True, if the properties are met, false otherwise.
- */
- public boolean isMetBy(GlobalProperties props) {
- if (this.partitioning == PartitioningProperty.ANY_DISTRIBUTION) {
- return true;
- } else if (this.partitioning == PartitioningProperty.FULL_REPLICATION) {
- return props.isFullyReplicated();
- }
- else if (props.isFullyReplicated()) {
- return false;
- }
- else if (this.partitioning == PartitioningProperty.RANDOM_PARTITIONED) {
- return true;
- }
- else if (this.partitioning == PartitioningProperty.ANY_PARTITIONING) {
- return checkCompatiblePartitioningFields(props);
- }
- else if (this.partitioning == PartitioningProperty.HASH_PARTITIONED) {
- return props.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- checkCompatiblePartitioningFields(props);
- }
- else if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
- return props.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
- props.matchesOrderedPartitioning(this.ordering);
- }
- else if (this.partitioning == PartitioningProperty.FORCED_REBALANCED) {
- return props.getPartitioning() == PartitioningProperty.FORCED_REBALANCED;
- }
- else if (this.partitioning == PartitioningProperty.CUSTOM_PARTITIONING) {
- return props.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
- checkCompatiblePartitioningFields(props) &&
- props.getCustomPartitioner().equals(this.customPartitioner);
-
- }
- else {
- throw new CompilerException("Properties matching logic leaves open cases.");
- }
- }
-
- /**
- * Parametrizes the ship strategy fields of a channel such that the channel produces
- * the desired global properties.
- *
- * @param channel The channel to parametrize.
- * @param globalDopChange Flag indicating whether the degree of parallelism changes
- * between sender and receiver.
- * @param exchangeMode The mode of data exchange (pipelined, always batch,
- * batch only on shuffle, ...)
- * @param breakPipeline Indicates whether this data exchange should break
- * pipelines (unless pipelines are forced).
- */
- public void parameterizeChannel(Channel channel, boolean globalDopChange,
- ExecutionMode exchangeMode, boolean breakPipeline) {
-
- // safety check. Fully replicated input must be preserved.
- if (channel.getSource().getGlobalProperties().isFullyReplicated() &&
- !(this.partitioning == PartitioningProperty.FULL_REPLICATION ||
- this.partitioning == PartitioningProperty.ANY_DISTRIBUTION))
- {
- throw new CompilerException("Fully replicated input must be preserved " +
- "and may not be converted into another global property.");
- }
-
- // if we request nothing, then we need no special strategy. forward, if the number of instances remains
- // the same, randomly repartition otherwise
- if (isTrivial() || this.partitioning == PartitioningProperty.ANY_DISTRIBUTION) {
- ShipStrategyType shipStrategy = globalDopChange ? ShipStrategyType.PARTITION_RANDOM :
- ShipStrategyType.FORWARD;
-
- DataExchangeMode em = DataExchangeMode.select(exchangeMode, shipStrategy, breakPipeline);
- channel.setShipStrategy(shipStrategy, em);
- return;
- }
-
- final GlobalProperties inGlobals = channel.getSource().getGlobalProperties();
- // if we have no global parallelism change, check if we have already compatible global properties
- if (!globalDopChange && isMetBy(inGlobals)) {
- DataExchangeMode em = DataExchangeMode.select(exchangeMode, ShipStrategyType.FORWARD, breakPipeline);
- channel.setShipStrategy(ShipStrategyType.FORWARD, em);
- return;
- }
-
- // if we fall through the conditions until here, we need to re-establish
- ShipStrategyType shipType;
- FieldList partitionKeys;
- boolean[] sortDirection;
- Partitioner<?> partitioner;
-
- switch (this.partitioning) {
- case FULL_REPLICATION:
- shipType = ShipStrategyType.BROADCAST;
- partitionKeys = null;
- sortDirection = null;
- partitioner = null;
- break;
-
- case ANY_PARTITIONING:
- case HASH_PARTITIONED:
- shipType = ShipStrategyType.PARTITION_HASH;
- partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
- sortDirection = null;
- partitioner = null;
- break;
-
- case RANGE_PARTITIONED:
- shipType = ShipStrategyType.PARTITION_RANGE;
- partitionKeys = this.ordering.getInvolvedIndexes();
- sortDirection = this.ordering.getFieldSortDirections();
- partitioner = null;
-
- if (this.dataDistribution != null) {
- channel.setDataDistribution(this.dataDistribution);
- }
- break;
-
- case FORCED_REBALANCED:
- shipType = ShipStrategyType.PARTITION_FORCED_REBALANCE;
- partitionKeys = null;
- sortDirection = null;
- partitioner = null;
- break;
-
- case CUSTOM_PARTITIONING:
- shipType = ShipStrategyType.PARTITION_CUSTOM;
- partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
- sortDirection = null;
- partitioner = this.customPartitioner;
- break;
-
- default:
- throw new CompilerException("Invalid partitioning to create through a data exchange: "
- + this.partitioning.name());
- }
-
- DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
- channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
- }
-
- // ------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((partitioning == null) ? 0 : partitioning.ordinal());
- result = prime * result + ((partitioningFields == null) ? 0 : partitioningFields.hashCode());
- result = prime * result + ((ordering == null) ? 0 : ordering.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj != null && obj instanceof RequestedGlobalProperties) {
- RequestedGlobalProperties other = (RequestedGlobalProperties) obj;
- return (ordering == other.getOrdering() || (ordering != null && ordering.equals(other.getOrdering())))
- && (partitioning == other.getPartitioning())
- && (partitioningFields == other.partitioningFields ||
- (partitioningFields != null && partitioningFields.equals(other.getPartitionedFields())));
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "Requested Global Properties [partitioning=" + partitioning +
- (this.partitioningFields == null ? "" : ", on fields " + this.partitioningFields) +
- (this.ordering == null ? "" : ", with ordering " + this.ordering) + "]";
- }
-
- public RequestedGlobalProperties clone() {
- try {
- return (RequestedGlobalProperties) super.clone();
- } catch (CloneNotSupportedException cnse) {
- // should never happen, but propagate just in case
- throw new RuntimeException(cnse);
- }
- }
-
- private boolean checkCompatiblePartitioningFields(GlobalProperties props) {
- if(this.partitioningFields instanceof FieldList) {
- // partitioningFields as FieldList requires strict checking!
- return props.isExactlyPartitionedOnFields((FieldList)this.partitioningFields);
- } else {
- return props.isPartitionedOnFields(this.partitioningFields);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
deleted file mode 100644
index a9d14e8..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedLocalProperties.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.dataproperties;
-
-import java.util.Arrays;
-
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-
-/**
- * This class represents the local properties of the data that are requested by an operator.
- * Local properties are the properties within one partition.
- * Operators request the local properties they need for correct execution. Here are some example local
- * properties requested by certain operators:
- * <ul>
- * <li>"groupBy/reduce" will request the data to be grouped on the key fields.</li>
- * <li>A sort-merge join will request the data from each input to be sorted on the respective join key.</li>
- * </ul>
- */
-public class RequestedLocalProperties implements Cloneable {
-
- private Ordering ordering; // order inside a partition, null if not ordered
-
- private FieldSet groupedFields; // fields by which the stream is grouped. null if not grouped.
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Default constructor for trivial local properties. No order, no grouping, no uniqueness.
- */
- public RequestedLocalProperties() {}
-
- /**
- * Creates interesting properties for the given ordering.
- *
- * @param ordering The interesting ordering.
- */
- public RequestedLocalProperties(Ordering ordering) {
- this.ordering = ordering;
- }
-
- /**
- * Creates interesting properties for the given grouping.
- *
- * @param groupedFields The set of fields whose grouping is interesting.
- */
- public RequestedLocalProperties(FieldSet groupedFields) {
- this.groupedFields = groupedFields;
- }
-
- /**
- * This constructor is used only for internal copy creation.
- *
- * @param ordering The ordering represented by these local properties.
- * @param groupedFields The grouped fields for these local properties.
- */
- private RequestedLocalProperties(Ordering ordering, FieldSet groupedFields) {
- this.ordering = ordering;
- this.groupedFields = groupedFields;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the key order.
- *
- * @return The key order, or <code>null</code> if nothing is ordered.
- */
- public Ordering getOrdering() {
- return ordering;
- }
-
- /**
- * Sets the order for these interesting local properties.
- *
- * @param ordering The order to set.
- */
- public void setOrdering(Ordering ordering) {
- this.ordering = ordering;
- }
-
- /**
- * Gets the grouped fields.
- *
- * @return The grouped fields, or <code>null</code> if nothing is grouped.
- */
- public FieldSet getGroupedFields() {
- return this.groupedFields;
- }
-
- /**
- * Sets the fields that are grouped in these data properties.
- *
- * @param groupedFields The fields that are grouped in these data properties.
- */
- public void setGroupedFields(FieldSet groupedFields) {
- this.groupedFields = groupedFields;
- }
-
- /**
- * Checks, if the properties in this object are trivial, i.e. only standard values.
- */
- public boolean isTrivial() {
- return ordering == null && this.groupedFields == null;
- }
-
- /**
- * This method resets the local properties to a state where no properties are given.
- */
- public void reset() {
- this.ordering = null;
- this.groupedFields = null;
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Filters these properties by what can be preserved by the given SemanticProperties when propagated down
- * to the given input.
- *
- * @param props The SemanticProperties which define which fields are preserved.
- * @param input The index of the operator's input.
- * @return The filtered RequestedLocalProperties
- */
- public RequestedLocalProperties filterBySemanticProperties(SemanticProperties props, int input) {
-
- // no semantic properties, all local properties are filtered
- if (props == null) {
- throw new NullPointerException("SemanticProperties may not be null.");
- }
-
- if (this.ordering != null) {
- Ordering newOrdering = new Ordering();
-
- for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
- int targetField = this.ordering.getInvolvedIndexes().get(i);
- int sourceField = props.getForwardingSourceField(input, targetField);
- if (sourceField >= 0) {
- newOrdering.appendOrdering(sourceField, this.ordering.getType(i), this.ordering.getOrder(i));
- } else {
- return null;
- }
- }
- return new RequestedLocalProperties(newOrdering);
-
- } else if (this.groupedFields != null) {
- FieldSet newGrouping = new FieldSet();
-
- // check, whether the local key grouping is preserved
- for (Integer targetField : this.groupedFields) {
- int sourceField = props.getForwardingSourceField(input, targetField);
- if (sourceField >= 0) {
- newGrouping = newGrouping.addField(sourceField);
- } else {
- return null;
- }
- }
- return new RequestedLocalProperties(newGrouping);
- } else {
- return null;
- }
- }
-
- /**
- * Checks, if this set of properties, as interesting properties, is met by the given
- * properties.
- *
- * @param other
- * The properties for which to check whether they meet these properties.
- * @return True, if the properties are met, false otherwise.
- */
- public boolean isMetBy(LocalProperties other) {
- if (this.ordering != null) {
- // we demand an ordering
- return other.getOrdering() != null && this.ordering.isMetBy(other.getOrdering());
- } else if (this.groupedFields != null) {
- // check if the other fields are unique
- if (other.getGroupedFields() != null && other.getGroupedFields().isValidUnorderedPrefix(this.groupedFields)) {
- return true;
- } else {
- return other.areFieldsUnique(this.groupedFields);
- }
- } else {
- return true;
- }
- }
-
- /**
- * Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties.
- *
- * @param channel The channel to parametrize.
- */
- public void parameterizeChannel(Channel channel) {
- LocalProperties current = channel.getLocalProperties();
-
- if (isMetBy(current)) {
- // we are met, all is good
- channel.setLocalStrategy(LocalStrategy.NONE);
- }
- else if (this.ordering != null) {
- channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections());
- }
- else if (this.groupedFields != null) {
- boolean[] dirs = new boolean[this.groupedFields.size()];
- Arrays.fill(dirs, true);
- channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs);
- }
- else {
- channel.setLocalStrategy(LocalStrategy.NONE);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (this.ordering == null ? 0 : this.ordering.hashCode());
- result = prime * result + (this.groupedFields == null ? 0 : this.groupedFields.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof RequestedLocalProperties) {
- final RequestedLocalProperties other = (RequestedLocalProperties) obj;
- return (this.ordering == other.ordering || (this.ordering != null && this.ordering.equals(other.ordering))) &&
- (this.groupedFields == other.groupedFields || (this.groupedFields != null && this.groupedFields.equals(other.groupedFields)));
- } else {
- return false;
- }
- }
-
- @Override
- public String toString() {
- return "Requested Local Properties [ordering=" + this.ordering + ", grouped=" + this.groupedFields + "]";
- }
-
- @Override
- public RequestedLocalProperties clone() {
- return new RequestedLocalProperties(this.ordering, this.groupedFields);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
deleted file mode 100644
index 707142f..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockEdge.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.deadlockdetect;
-
-public class DeadlockEdge {
-
- private DeadlockVertex destination;
-
- public DeadlockEdge( DeadlockVertex d ){
- destination = d;
- }
-
- public DeadlockVertex getDestination() {
- return destination;
- }
-
- public void setDestination(DeadlockVertex destination) {
- this.destination = destination;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
deleted file mode 100644
index cae75ad..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockGraph.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.deadlockdetect;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Set;
-
-import org.apache.flink.optimizer.plan.PlanNode;
-
-public class DeadlockGraph {
-
- public Set<DeadlockVertex> vertices;
-
- public DeadlockGraph() {
- this.vertices = new HashSet<DeadlockVertex>();
- }
-
- public Set<DeadlockVertex> vertices() {
- return vertices;
- }
-
- public DeadlockVertex addVertex(PlanNode original) {
-
- DeadlockVertex v = new DeadlockVertex(original);
- this.vertices.add(v);
- return v;
- }
-
- public void addEdge(PlanNode source, PlanNode destination) {
-
- DeadlockVertex dest = null;
- for(DeadlockVertex v : vertices) {
- if(v.getOriginal().equals(destination)) {
- dest = v;
- }
- }
-
- for(DeadlockVertex v : vertices) {
- if(v.getOriginal().equals(source)) {
- v.addEdge(dest);
- }
- }
-
- }
-
- public long size() {
- return vertices.size();
- }
-
- public String toString() {
- StringBuilder out = new StringBuilder();
- out.append("------------ GRAPH ------------\n");
- for (DeadlockVertex n : vertices) {
- out.append("Node " +n+"_\n");
- for(DeadlockEdge a: n.getOutEdges()) {
- out.append("\t->"+a.getDestination()+"\n");
- }
- out.append("\n");
- }
-
- return out.toString();
- }
-
- public boolean hasCycle() {
-
- Collection <DeadlockVertex> vertexCollect = this.vertices();
-
- Queue <DeadlockVertex> q; // Queue will store vertices that have in-degree of zero
-
- // Calculate the in-degree of all vertices
- for (DeadlockVertex v: vertexCollect) {
- v.setInDegree(0);
- }
-
- for (DeadlockVertex v: vertexCollect) {
- for(DeadlockEdge edge : v.getOutEdges()) {
- edge.getDestination().setInDegree(edge.getDestination().getInDegree()+1);
- }
- }
-
- // Find all vertices with in-degree == 0 and put in queue
- q = new LinkedList<DeadlockVertex>();
- for (DeadlockVertex v : vertexCollect) {
- if (v.getInDegree() == 0) {
- q.offer(v);
- }
- }
-
- while (!q.isEmpty()) {
-
- DeadlockVertex v = q.poll();
- this.vertices.remove(v);
-
- for (DeadlockEdge e: v.getOutEdges()) {
-
- DeadlockVertex w = e.getDestination();
- w.setInDegree(w.getInDegree() - 1);
-
- if(w.getInDegree() == 0) {
- q.offer(w);
- }
- }
- }
-
- if (!vertexCollect.isEmpty() ){
- return true; //Cycle found
- }
-
- return false;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
deleted file mode 100644
index 19ee6c5..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockPreventer.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.deadlockdetect;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * Certain pipelined flows may lead to deadlocks, in which case we need to make sure the pipelines are broken or made elastic enough to prevent that.
- *
- * This is only relevat to pipelined data flows where one operator has more than one consumers (successors in the flow).
- *
- * Most cases are caught by the general logic that deals with branching/joining flows. The following cases need additional checks:
- *
- * <build>
- * (source1) ------ (join)
- * \ / <probe>
- * \ /
- * \/
- * /\
- * / \
- * / \ <probe>
- * (source2) ------(join)
- * <build>
- *
- * Since both sources pipeline their data into a build and a probe side, they get stalled by the back pressure from the probe side (which waits for the build side to complete) and never finish the build side.
- *
- * We can model dependencies of pipelined / materialized connections and do a reguar cyclic dependencies check to detect such situations. Pipelined connections have a dependency from sender to receiver, non-pipelined (fully dammed) connections have a dependency from receiver to sender.
- *
- */
-public class DeadlockPreventer implements Visitor<PlanNode> {
-
- private DeadlockGraph g;
-
- public DeadlockPreventer() {
- this.g = new DeadlockGraph();
- }
-
- public void resolveDeadlocks(List<? extends PlanNode> sinks) {
-
- for(PlanNode s : sinks) {
- s.accept(this);
- }
- if(g.hasCycle()) {
-
- // in the remaining plan is a cycle
- for(DeadlockVertex v : g.vertices) {
-
- // first strategy to fix -> swap build and probe side
- if(v.getOriginal().getDriverStrategy().equals(DriverStrategy.HYBRIDHASH_BUILD_FIRST)) {
-
- v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
-
- if(hasDeadlock(sinks)) {
- // Didn't fix anything -> revert
- v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
- }
- else {
- // deadlock resolved
- break;
- }
- }
-
- // other direction
- if(v.getOriginal().getDriverStrategy().equals(DriverStrategy.HYBRIDHASH_BUILD_SECOND)) {
-
- v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_FIRST);
-
- if(hasDeadlock(sinks)) {
- // Didn't fix anything -> revert
- v.getOriginal().setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
- }
- else {
- // deadlock resolved
- break;
- }
- }
- }
-
- // switching build and probe side did not help -> pipeline breaker
- for(DeadlockVertex v : g.vertices) {
- if(v.getOriginal() instanceof DualInputPlanNode) {
- DualInputPlanNode n = (DualInputPlanNode) v.getOriginal();
-
- // what is the pipelined side? (other side should be a dam, otherwise operator could not be source of deadlock)
- if(!(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())
- && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
- n.getInput1().setTempMode(n.getInput1().getTempMode().makePipelineBreaker());
- }
- else if( !(n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())
- && (n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())) {
- n.getInput2().setTempMode(n.getInput2().getTempMode().makePipelineBreaker());
- }
-
- // Deadlock resolved?
- if(!hasDeadlock(sinks)) {
- break;
- }
- }
- }
- }
-
- }
-
- /**
- * Creates new DeadlockGraph from plan and checks for cycles
- */
- public boolean hasDeadlock(List<? extends PlanNode> sinks) {
- this.g = new DeadlockGraph();
-
- for(PlanNode s : sinks) {
- s.accept(this);
- }
-
- if(g.hasCycle()) {
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public boolean preVisit(PlanNode visitable) {
-
- g.addVertex(visitable);
- return true;
- }
-
- @Override
- public void postVisit(PlanNode visitable) {
- if(visitable instanceof SingleInputPlanNode) {
- SingleInputPlanNode n = (SingleInputPlanNode) visitable;
-
- if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput().getLocalStrategy().dams() || n.getInput().getTempMode().breaksPipeline()) {
- g.addEdge(n, n.getPredecessor());
- }
- else {
- g.addEdge(n.getPredecessor(), n);
- }
- }
- else if(visitable instanceof DualInputPlanNode) {
- DualInputPlanNode n = (DualInputPlanNode) visitable;
-
- if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline()) {
- g.addEdge(n, n.getInput1().getSource());
- }
- else {
- g.addEdge(n.getInput1().getSource(), n);
- }
-
- if(!n.getDriverStrategy().equals(DriverStrategy.NONE) && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
- g.addEdge(n, n.getInput2().getSource());
- }
- else {
- g.addEdge(n.getInput2().getSource(), n);
- }
- }
-
-
- // recursively fix iterations
- if (visitable instanceof BulkIterationPlanNode) {
-
- DeadlockPreventer dp = new DeadlockPreventer();
- List<PlanNode> planSinks = new ArrayList<PlanNode>(1);
-
- BulkIterationPlanNode pspn = (BulkIterationPlanNode) visitable;
- planSinks.add(pspn.getRootOfStepFunction());
-
- dp.resolveDeadlocks(planSinks);
-
- }
- else if (visitable instanceof WorksetIterationPlanNode) {
-
- DeadlockPreventer dp = new DeadlockPreventer();
- List<PlanNode> planSinks = new ArrayList<PlanNode>(2);
-
- WorksetIterationPlanNode pspn = (WorksetIterationPlanNode) visitable;
- planSinks.add(pspn.getSolutionSetDeltaPlanNode());
- planSinks.add(pspn.getNextWorkSetPlanNode());
-
- dp.resolveDeadlocks(planSinks);
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
deleted file mode 100644
index f26763e..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/deadlockdetect/DeadlockVertex.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.deadlockdetect;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.optimizer.plan.PlanNode;
-
-public class DeadlockVertex {
-
- private PlanNode original;
-
- private List<DeadlockEdge> outEdges;
-
- private int inDegree;
-
- public DeadlockVertex( PlanNode original ) {
- this.original = original;
- outEdges = new LinkedList<DeadlockEdge>();
- inDegree = 0;
- }
-
- public void addEdge(DeadlockVertex destination) {
-
- // no duplicates
- for(DeadlockEdge e : outEdges) {
- if(e.getDestination().equals(destination)) {
- return;
- }
- }
-
- DeadlockEdge e = new DeadlockEdge(destination);
- this.outEdges.add(e);
- }
-
- public boolean equals(Object o) {
-
- if(!(o instanceof DeadlockVertex)) {
- return false;
- }
-
- DeadlockVertex v = (DeadlockVertex) o;
- if(v.getOriginal().equals(this.getOriginal())) {
- return true;
- }
-
- return false;
- }
-
- public int hashCode() {
- return this.original.hashCode();
- }
-
- public String toString() {
- return original.toString();
- }
- public PlanNode getOriginal() {
- return original;
- }
-
- public void setOriginal(PlanNode original) {
- this.original = original;
- }
-
- public int getInDegree() {
- return inDegree;
- }
-
- public void setInDegree(int inDegree) {
- this.inDegree = inDegree;
- }
-
- public List<DeadlockEdge> getOutEdges() {
- return outEdges;
- }
-
- public void setOutEdges(List<DeadlockEdge> outEdges) {
- this.outEdges = outEdges;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
deleted file mode 100644
index 2b634ae..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-
-/**
- * Defines the possible global properties for a join.
- */
-public abstract class AbstractJoinDescriptor extends OperatorDescriptorDual {
-
- private final boolean broadcastFirstAllowed;
- private final boolean broadcastSecondAllowed;
- private final boolean repartitionAllowed;
-
- private Partitioner<?> customPartitioner;
-
- protected AbstractJoinDescriptor(FieldList keys1, FieldList keys2) {
- this(keys1, keys2, true, true, true);
- }
-
- protected AbstractJoinDescriptor(FieldList keys1, FieldList keys2,
- boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
- {
- super(keys1, keys2);
-
- this.broadcastFirstAllowed = broadcastFirstAllowed;
- this.broadcastSecondAllowed = broadcastSecondAllowed;
- this.repartitionAllowed = repartitionAllowed;
- }
-
- public void setCustomPartitioner(Partitioner<?> partitioner) {
- customPartitioner = partitioner;
- }
-
- @Override
- protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
- ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
-
- if (repartitionAllowed) {
- // partition both (hash or custom)
- if (this.customPartitioner == null) {
-
- // we accept compatible partitionings of any type
- RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
- RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
- partitioned_left_any.setAnyPartitioning(this.keys1);
- partitioned_right_any.setAnyPartitioning(this.keys2);
- pairs.add(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any));
-
- // add strict hash partitioning of both inputs on their full key sets
- RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
- RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
- partitioned_left_hash.setHashPartitioned(this.keys1);
- partitioned_right_hash.setHashPartitioned(this.keys2);
- pairs.add(new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
- }
- else {
- RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
- partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);
-
- RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
- partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);
-
- return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
- }
-
-
- RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
- if (customPartitioner == null) {
- partitioned1.setAnyPartitioning(this.keys1);
- } else {
- partitioned1.setCustomPartitioned(this.keys1, this.customPartitioner);
- }
-
- RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
- if (customPartitioner == null) {
- partitioned2.setAnyPartitioning(this.keys2);
- } else {
- partitioned2.setCustomPartitioned(this.keys2, this.customPartitioner);
- }
-
- pairs.add(new GlobalPropertiesPair(partitioned1, partitioned2));
- }
-
- if (broadcastSecondAllowed) {
- // replicate second
- RequestedGlobalProperties any1 = new RequestedGlobalProperties();
- RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
- replicated2.setFullyReplicated();
- pairs.add(new GlobalPropertiesPair(any1, replicated2));
- }
-
- if (broadcastFirstAllowed) {
- // replicate first
- RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
- replicated1.setFullyReplicated();
- RequestedGlobalProperties any2 = new RequestedGlobalProperties();
- pairs.add(new GlobalPropertiesPair(replicated1, any2));
- }
- return pairs;
- }
-
- @Override
- public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
- GlobalProperties produced1, GlobalProperties produced2)
- {
- if (requested1.getPartitioning().isPartitionedOnKey() && requested2.getPartitioning().isPartitionedOnKey()) {
-
- if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {
-
- // both are hash partitioned, check that partitioning fields are equivalently chosen
- return checkEquivalentFieldPositionsInKeyFields(
- produced1.getPartitioningFields(), produced2.getPartitioningFields());
-
- }
- else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
-
- // both are range partitioned, check that partitioning fields are equivalently chosen
- return checkEquivalentFieldPositionsInKeyFields(
- produced1.getPartitioningFields(), produced2.getPartitioningFields());
-
- }
- else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
- produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {
-
- // both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
- return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
- produced2.getPartitioningFields().isExactMatch(this.keys2) &&
- produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
- produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());
-
- }
- else {
-
- // no other partitioning valid, incl. ANY_PARTITIONING.
- // For joins we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
- return false;
- }
-
- } else {
- return true;
- }
-
- }
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
- GlobalProperties gp = GlobalProperties.combine(in1, in2);
- if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
- gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gp.clearUniqueFieldCombinations();
- return gp;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
deleted file mode 100644
index 8125556..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AbstractOperatorDescriptor.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- * The root of all descriptions of an operator to the optimizer.
- */
-public interface AbstractOperatorDescriptor {
-
- /**
- * Gets the runtime strategy to be executed by this operator.
- *
- * @return The runtime strategy.
- */
- DriverStrategy getStrategy();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
deleted file mode 100644
index 9ee56b0..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupCombineProperties.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-import java.util.Collections;
-import java.util.List;
-
-public final class AllGroupCombineProperties extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.ALL_GROUP_COMBINE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupCombine ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_COMBINE);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
deleted file mode 100644
index 9efd8c7..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/operators/AllGroupReduceProperties.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.operators;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
-import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
-import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-public final class AllGroupReduceProperties extends OperatorDescriptorSingle {
-
- @Override
- public DriverStrategy getStrategy() {
- return DriverStrategy.ALL_GROUP_REDUCE;
- }
-
- @Override
- public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
- return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
- }
-
- @Override
- protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
- return Collections.singletonList(new RequestedGlobalProperties());
- }
-
- @Override
- protected List<RequestedLocalProperties> createPossibleLocalProperties() {
- return Collections.singletonList(new RequestedLocalProperties());
- }
-
-
- @Override
- public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
- if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
- gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
- {
- gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
- }
- gProps.clearUniqueFieldCombinations();
- return gProps;
- }
-
-
- @Override
- public LocalProperties computeLocalProperties(LocalProperties lProps) {
- return lProps.clearUniqueFieldSets();
- }
-}
[32/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
deleted file mode 100644
index 6f634fb..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/PlanNode.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.OptimizerNode.UnclosedBranchDescriptor;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plandump.DumpableConnection;
-import org.apache.flink.optimizer.plandump.DumpableNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.util.Visitable;
-
-/**
- * The representation of a data exchange between to operators. The data exchange can realize a shipping strategy,
- * which established global properties, and a local strategy, which establishes local properties.
- * <p>
- * Because we currently deal only with plans where the operator order is fixed, many properties are equal
- * among candidates and are determined prior to the enumeration (such as for example constant/dynamic path membership).
- * Hence, many methods will delegate to the {@code OptimizerNode} that represents the node this candidate was
- * created for.
- */
-public abstract class PlanNode implements Visitable<PlanNode>, DumpableNode<PlanNode> {
-
- protected final OptimizerNode template;
-
- protected final List<Channel> outChannels;
-
- private List<NamedChannel> broadcastInputs;
-
- private final String nodeName;
-
- private DriverStrategy driverStrategy; // The local strategy (sorting / hashing, ...)
-
- protected LocalProperties localProps; // local properties of the data produced by this node
-
- protected GlobalProperties globalProps; // global properties of the data produced by this node
-
- protected Map<OptimizerNode, PlanNode> branchPlan; // the actual plan alternative chosen at a branch point
-
- protected Costs nodeCosts; // the costs incurred by this node
-
- protected Costs cumulativeCosts; // the cumulative costs of all operators in the sub-tree
-
- private double relativeMemoryPerSubTask; // the amount of memory dedicated to each task, in bytes
-
- private int parallelism;
-
- private boolean pFlag; // flag for the internal pruning algorithm
-
- // --------------------------------------------------------------------------------------------
-
- public PlanNode(OptimizerNode template, String nodeName, DriverStrategy strategy) {
- this.outChannels = new ArrayList<Channel>(2);
- this.broadcastInputs = new ArrayList<NamedChannel>();
- this.template = template;
- this.nodeName = nodeName;
- this.driverStrategy = strategy;
-
- this.parallelism = template.getParallelism();
-
- // check, if there is branch at this node. if yes, this candidate must be associated with
- // the branching template node.
- if (template.isBranching()) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
- this.branchPlan.put(template, this);
- }
- }
-
- protected void mergeBranchPlanMaps(PlanNode pred1, PlanNode pred2) {
- mergeBranchPlanMaps(pred1.branchPlan, pred2.branchPlan);
- }
-
- protected void mergeBranchPlanMaps(Map<OptimizerNode, PlanNode> branchPlan1, Map<OptimizerNode, PlanNode> branchPlan2) {
- // merge the branchPlan maps according the template's uncloseBranchesStack
- if (this.template.hasUnclosedBranches()) {
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
- }
-
- for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
- OptimizerNode brancher = uc.getBranchingNode();
- PlanNode selectedCandidate = null;
-
- if (branchPlan1 != null) {
- // predecessor 1 has branching children, see if it got the branch we are looking for
- selectedCandidate = branchPlan1.get(brancher);
- }
-
- if (selectedCandidate == null && branchPlan2 != null) {
- // predecessor 2 has branching children, see if it got the branch we are looking for
- selectedCandidate = branchPlan2.get(brancher);
- }
-
- // it may be that the branch candidate is only found once the broadcast variables are set
- if (selectedCandidate != null) {
- this.branchPlan.put(brancher, selectedCandidate);
- }
- }
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Accessors
- // --------------------------------------------------------------------------------------------
-
- /**
- * Gets the node from the optimizer DAG for which this plan candidate node was created.
- *
- * @return The optimizer's DAG node.
- */
- public OptimizerNode getOriginalOptimizerNode() {
- return this.template;
- }
-
- /**
- * Gets the program operator that this node represents in the plan.
- *
- * @return The program operator this node represents in the plan.
- */
- public Operator<?> getProgramOperator() {
- return this.template.getOperator();
- }
-
- /**
- * Gets the name of the plan node.
- *
- * @return The name of the plan node.
- */
- public String getNodeName() {
- return this.nodeName;
- }
-
- public int getMemoryConsumerWeight() {
- return this.driverStrategy.isMaterializing() ? 1 : 0;
- }
-
- /**
- * Gets the memory dedicated to each sub-task for this node.
- *
- * @return The memory per task, in bytes.
- */
- public double getRelativeMemoryPerSubTask() {
- return this.relativeMemoryPerSubTask;
- }
-
- /**
- * Sets the memory dedicated to each task for this node.
- *
- * @param relativeMemoryPerSubtask The relative memory per sub-task
- */
- public void setRelativeMemoryPerSubtask(double relativeMemoryPerSubtask) {
- this.relativeMemoryPerSubTask = relativeMemoryPerSubtask;
- }
-
- /**
- * Gets the driver strategy from this node. This determines for example for a <i>match</i> Pact whether
- * to use a merge or a hybrid hash strategy.
- *
- * @return The driver strategy.
- */
- public DriverStrategy getDriverStrategy() {
- return this.driverStrategy;
- }
-
- /**
- * Sets the driver strategy for this node. Usually should not be changed.
- *
- * @param newDriverStrategy The driver strategy.
- */
- public void setDriverStrategy(DriverStrategy newDriverStrategy) {
- this.driverStrategy = newDriverStrategy;
- }
-
- public void initProperties(GlobalProperties globals, LocalProperties locals) {
- if (this.globalProps != null || this.localProps != null) {
- throw new IllegalStateException();
- }
- this.globalProps = globals;
- this.localProps = locals;
- }
-
- /**
- * Gets the local properties from this PlanNode.
- *
- * @return The local properties.
- */
- public LocalProperties getLocalProperties() {
- return this.localProps;
- }
-
- /**
- * Gets the global properties from this PlanNode.
- *
- * @return The global properties.
- */
- public GlobalProperties getGlobalProperties() {
- return this.globalProps;
- }
-
- /**
- * Gets the costs incurred by this node. The costs reflect also the costs incurred by the shipping strategies
- * of the incoming connections.
- *
- * @return The node-costs, or null, if not yet set.
- */
- public Costs getNodeCosts() {
- return this.nodeCosts;
- }
-
- /**
- * Gets the cumulative costs of this nose. The cumulative costs are the sum of the costs
- * of this node and of all nodes in the subtree below this node.
- *
- * @return The cumulative costs, or null, if not yet set.
- */
- public Costs getCumulativeCosts() {
- return this.cumulativeCosts;
- }
-
- public Costs getCumulativeCostsShare() {
- if (this.cumulativeCosts == null) {
- return null;
- } else {
- Costs result = cumulativeCosts.clone();
- if (this.template.getOutgoingConnections() != null) {
- int outDegree = this.template.getOutgoingConnections().size();
- if (outDegree > 0) {
- result.divideBy(outDegree);
- }
- }
-
- return result;
- }
- }
-
-
- /**
- * Sets the basic cost for this node to the given value, and sets the cumulative costs
- * to those costs plus the cost shares of all inputs (regular and broadcast).
- *
- * @param nodeCosts The already knows costs for this node
- * (this cost a produces by a concrete {@code OptimizerNode} subclass.
- */
- public void setCosts(Costs nodeCosts) {
- // set the node costs
- this.nodeCosts = nodeCosts;
-
- // the cumulative costs are the node costs plus the costs of all inputs
- this.cumulativeCosts = nodeCosts.clone();
-
- // add all the normal inputs
- for (PlanNode pred : getPredecessors()) {
-
- Costs parentCosts = pred.getCumulativeCostsShare();
- if (parentCosts != null) {
- this.cumulativeCosts.addCosts(parentCosts);
- } else {
- throw new CompilerException("Trying to set the costs of an operator before the predecessor costs are computed.");
- }
- }
-
- // add all broadcast variable inputs
- if (this.broadcastInputs != null) {
- for (NamedChannel nc : this.broadcastInputs) {
- Costs bcInputCost = nc.getSource().getCumulativeCostsShare();
- if (bcInputCost != null) {
- this.cumulativeCosts.addCosts(bcInputCost);
- } else {
- throw new CompilerException("Trying to set the costs of an operator before the broadcast input costs are computed.");
- }
- }
- }
- }
-
- public void setParallelism(int parallelism) {
- this.parallelism = parallelism;
- }
-
- public int getParallelism() {
- return this.parallelism;
- }
-
- public long getGuaranteedAvailableMemory() {
- return this.template.getMinimalMemoryAcrossAllSubTasks();
- }
-
- public Map<OptimizerNode, PlanNode> getBranchPlan() {
- return branchPlan;
- }
-
- // --------------------------------------------------------------------------------------------
- // Input, Predecessors, Successors
- // --------------------------------------------------------------------------------------------
-
- public abstract Iterable<Channel> getInputs();
-
- @Override
- public abstract Iterable<PlanNode> getPredecessors();
-
- /**
- * Sets a list of all broadcast inputs attached to this node.
- */
- public void setBroadcastInputs(List<NamedChannel> broadcastInputs) {
- if (broadcastInputs != null) {
- this.broadcastInputs = broadcastInputs;
-
- // update the branch map
- for (NamedChannel nc : broadcastInputs) {
- PlanNode source = nc.getSource();
-
- mergeBranchPlanMaps(branchPlan, source.branchPlan);
- }
- }
-
- // do a sanity check that if we are branching, we have now candidates for each branch point
- if (this.template.hasUnclosedBranches()) {
- if (this.branchPlan == null) {
- throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
- }
-
- for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
- OptimizerNode brancher = uc.getBranchingNode();
- if (this.branchPlan.get(brancher) == null) {
- throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
- }
- }
- }
- }
-
- /**
- * Gets a list of all broadcast inputs attached to this node.
- */
- public List<NamedChannel> getBroadcastInputs() {
- return this.broadcastInputs;
- }
-
- /**
- * Adds a channel to a successor node to this node.
- *
- * @param channel The channel to the successor.
- */
- public void addOutgoingChannel(Channel channel) {
- this.outChannels.add(channel);
- }
-
- /**
- * Gets a list of all outgoing channels leading to successors.
- *
- * @return A list of all channels leading to successors.
- */
- public List<Channel> getOutgoingChannels() {
- return this.outChannels;
- }
-
- // --------------------------------------------------------------------------------------------
- // Miscellaneous
- // --------------------------------------------------------------------------------------------
-
- public void updatePropertiesWithUniqueSets(Set<FieldSet> uniqueFieldCombinations) {
- if (uniqueFieldCombinations == null || uniqueFieldCombinations.isEmpty()) {
- return;
- }
- for (FieldSet fields : uniqueFieldCombinations) {
- this.globalProps.addUniqueFieldCombination(fields);
- this.localProps = this.localProps.addUniqueFields(fields);
- }
- }
-
- public PlanNode getCandidateAtBranchPoint(OptimizerNode branchPoint) {
- if (branchPlan == null) {
- return null;
- } else {
- return this.branchPlan.get(branchPoint);
- }
- }
-
- /**
- * Sets the pruning marker to true.
- */
- public void setPruningMarker() {
- this.pFlag = true;
- }
-
- /**
- * Checks whether the pruning marker was set.
- *
- * @return True, if the pruning marker was set, false otherwise.
- */
- public boolean isPruneMarkerSet() {
- return this.pFlag;
- }
-
- public boolean isOnDynamicPath() {
- return this.template.isOnDynamicPath();
- }
-
- public int getCostWeight() {
- return this.template.getCostWeight();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Checks whether this node has a dam on the way down to the given source node. This method
- * returns either that (a) the source node is not found as a (transitive) child of this node,
- * (b) the node is found, but no dam is on the path, or (c) the node is found and a dam is on
- * the path.
- *
- * @param source The node on the path to which the dam is sought.
- * @return The result whether the node is found and whether a dam is on the path.
- */
- public abstract SourceAndDamReport hasDamOnPathDownTo(PlanNode source);
-
- public FeedbackPropertiesMeetRequirementsReport checkPartialSolutionPropertiesMet(PlanNode partialSolution, GlobalProperties feedbackGlobal, LocalProperties feedbackLocal) {
- if (this == partialSolution) {
- return FeedbackPropertiesMeetRequirementsReport.PENDING;
- }
-
- boolean found = false;
- boolean allMet = true;
- boolean allLocallyMet = true;
-
- for (Channel input : getInputs()) {
- FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
-
- if (inputState == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
- continue;
- }
- else if (inputState == FeedbackPropertiesMeetRequirementsReport.MET) {
- found = true;
- continue;
- }
- else if (inputState == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
- return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
- }
- else {
- found = true;
-
- // the partial solution was on the path here. check whether the channel requires
- // certain properties that are met, or whether the channel introduces new properties
-
- // if the plan introduces new global properties, then we can stop looking whether
- // the feedback properties are sufficient to meet the requirements
- if (input.getShipStrategy() != ShipStrategyType.FORWARD && input.getShipStrategy() != ShipStrategyType.NONE) {
- continue;
- }
-
- // first check whether this channel requires something that is not met
- if (input.getRequiredGlobalProps() != null && !input.getRequiredGlobalProps().isMetBy(feedbackGlobal)) {
- return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
- }
-
- // in general, not everything is met here already
- allMet = false;
-
- // if the plan introduces new local properties, we can stop checking for matching local properties
- if (inputState != FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET) {
-
- if (input.getLocalStrategy() == LocalStrategy.NONE) {
-
- if (input.getRequiredLocalProps() != null && !input.getRequiredLocalProps().isMetBy(feedbackLocal)) {
- return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
- }
-
- allLocallyMet = false;
- }
- }
- }
- }
-
- if (!found) {
- return FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION;
- } else if (allMet) {
- return FeedbackPropertiesMeetRequirementsReport.MET;
- } else if (allLocallyMet) {
- return FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET;
- } else {
- return FeedbackPropertiesMeetRequirementsReport.PENDING;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public String toString() {
- return this.template.getName() + " \"" + getProgramOperator().getName() + "\" : " + this.driverStrategy +
- " [[ " + this.globalProps + " ]] [[ " + this.localProps + " ]]";
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public OptimizerNode getOptimizerNode() {
- return this.template;
- }
-
- @Override
- public PlanNode getPlanNode() {
- return this;
- }
-
- @Override
- public Iterable<DumpableConnection<PlanNode>> getDumpableInputs() {
- List<DumpableConnection<PlanNode>> allInputs = new ArrayList<DumpableConnection<PlanNode>>();
-
- for (Channel c : getInputs()) {
- allInputs.add(c);
- }
-
- for (NamedChannel c : getBroadcastInputs()) {
- allInputs.add(c);
- }
-
- return allInputs;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public static enum SourceAndDamReport {
- NOT_FOUND, FOUND_SOURCE, FOUND_SOURCE_AND_DAM;
- }
-
-
-
- public static enum FeedbackPropertiesMeetRequirementsReport {
- /** Indicates that the path is irrelevant */
- NO_PARTIAL_SOLUTION,
-
- /** Indicates that the question whether the properties are met has been determined pending
- * dependent on global and local properties */
- PENDING,
-
- /** Indicates that the question whether the properties are met has been determined pending
- * dependent on global properties only */
- PENDING_LOCAL_MET,
-
- /** Indicates that the question whether the properties are met has been determined true */
- MET,
-
- /** Indicates that the question whether the properties are met has been determined false */
- NOT_MET;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
deleted file mode 100644
index b928be7..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SingleInputPlanNode.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.Visitor;
-
-/**
- *
- */
-public class SingleInputPlanNode extends PlanNode {
-
- protected final Channel input;
-
- protected final FieldList[] driverKeys;
-
- protected final boolean[][] driverSortOrders;
-
- private TypeComparatorFactory<?>[] comparators;
-
- public Object postPassHelper;
-
- // --------------------------------------------------------------------------------------------
-
- public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy) {
- this(template, nodeName, input, driverStrategy, null, null);
- }
-
- public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input,
- DriverStrategy driverStrategy, FieldList driverKeyFields)
- {
- this(template, nodeName, input, driverStrategy, driverKeyFields, getTrueArray(driverKeyFields.size()));
- }
-
- public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input,
- DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
- {
- super(template, nodeName, driverStrategy);
- this.input = input;
-
- this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
- this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
- this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
-
- if(driverStrategy.getNumRequiredComparators() > 0) {
- this.driverKeys[0] = driverKeyFields;
- this.driverSortOrders[0] = driverSortOrders;
- }
-
- if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
- this.input.setReplicationFactor(getParallelism());
- }
-
- final PlanNode predNode = input.getSource();
-
- if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
-
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
- }
- this.branchPlan.putAll(predNode.branchPlan);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public SingleInputNode getSingleInputNode() {
- if (this.template instanceof SingleInputNode) {
- return (SingleInputNode) this.template;
- } else {
- throw new RuntimeException();
- }
- }
-
- /**
- * Gets the input channel to this node.
- *
- * @return The input channel to this node.
- */
- public Channel getInput() {
- return this.input;
- }
-
- /**
- * Gets the predecessor of this node, i.e. the source of the input channel.
- *
- * @return The predecessor of this node.
- */
- public PlanNode getPredecessor() {
- return this.input.getSource();
- }
-
- /**
- * Sets the key field indexes for the specified driver comparator.
- *
- * @param keys The key field indexes for the specified driver comparator.
- * @param id The ID of the driver comparator.
- */
- public void setDriverKeyInfo(FieldList keys, int id) {
- this.setDriverKeyInfo(keys, getTrueArray(keys.size()), id);
- }
-
- /**
- * Sets the key field information for the specified driver comparator.
- *
- * @param keys The key field indexes for the specified driver comparator.
- * @param sortOrder The key sort order for the specified driver comparator.
- * @param id The ID of the driver comparator.
- */
- public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) {
- if(id < 0 || id >= driverKeys.length) {
- throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only "
- +super.getDriverStrategy().getNumRequiredComparators()+" comparators.");
- }
- this.driverKeys[id] = keys;
- this.driverSortOrders[id] = sortOrder;
- }
-
- /**
- * Gets the key field indexes for the specified driver comparator.
- *
- * @param id The id of the driver comparator for which the key field indexes are requested.
- * @return The key field indexes of the specified driver comparator.
- */
- public FieldList getKeys(int id) {
- return this.driverKeys[id];
- }
-
- /**
- * Gets the sort order for the specified driver comparator.
- *
- * @param id The id of the driver comparator for which the sort order is requested.
- * @return The sort order of the specified driver comparator.
- */
- public boolean[] getSortOrders(int id) {
- return driverSortOrders[id];
- }
-
- /**
- * Gets the specified comparator from this PlanNode.
- *
- * @param id The ID of the requested comparator.
- *
- * @return The specified comparator.
- */
- public TypeComparatorFactory<?> getComparator(int id) {
- return comparators[id];
- }
-
- /**
- * Sets the specified comparator for this PlanNode.
- *
- * @param comparator The comparator to set.
- * @param id The ID of the comparator to set.
- */
- public void setComparator(TypeComparatorFactory<?> comparator, int id) {
- this.comparators[id] = comparator;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- this.input.getSource().accept(visitor);
-
- for (Channel broadcastInput : getBroadcastInputs()) {
- broadcastInput.getSource().accept(visitor);
- }
-
- visitor.postVisit(this);
- }
- }
-
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) {
- return Collections.singleton(this.input.getSource());
- }
- else {
- List<PlanNode> preds = new ArrayList<PlanNode>();
- preds.add(input.getSource());
-
- for (Channel c : getBroadcastInputs()) {
- preds.add(c.getSource());
- }
-
- return preds;
- }
- }
-
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.singleton(this.input);
- }
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
- SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source);
- if (res == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (res == FOUND_SOURCE) {
- return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() ||
- getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
- FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
- }
- else {
- // NOT_FOUND
- // check the broadcast inputs
-
- for (NamedChannel nc : getBroadcastInputs()) {
- SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source);
- if (bcRes != NOT_FOUND) {
- // broadcast inputs are always dams
- return FOUND_SOURCE_AND_DAM;
- }
- }
- return NOT_FOUND;
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- protected static boolean[] getTrueArray(int length) {
- final boolean[] a = new boolean[length];
- for (int i = 0; i < length; i++) {
- a[i] = true;
- }
- return a;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
deleted file mode 100644
index 451484d..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkJoinerPlanNode.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import java.util.List;
-
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-
-/**
- *
- */
-public class SinkJoinerPlanNode extends DualInputPlanNode {
-
- public SinkJoinerPlanNode(SinkJoiner template, Channel input1, Channel input2) {
- super(template, "", input1, input2, DriverStrategy.BINARY_NO_OP);
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void setCosts(Costs nodeCosts) {
- // the plan enumeration logic works as for regular two-input-operators, which is important
- // because of the branch handling logic. it does pick redistributing network channels
- // between the sink and the sink joiner, because sinks joiner has a different DOP than the sink.
- // we discard any cost and simply use the sum of the costs from the two children.
-
- Costs totalCosts = getInput1().getSource().getCumulativeCosts().clone();
- totalCosts.addCosts(getInput2().getSource().getCumulativeCosts());
- super.setCosts(totalCosts);
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void getDataSinks(List<SinkPlanNode> sinks) {
- final PlanNode in1 = this.input1.getSource();
- final PlanNode in2 = this.input2.getSource();
-
- if (in1 instanceof SinkPlanNode) {
- sinks.add((SinkPlanNode) in1);
- } else if (in1 instanceof SinkJoinerPlanNode) {
- ((SinkJoinerPlanNode) in1).getDataSinks(sinks);
- } else {
- throw new CompilerException("Illegal child node for a sink joiner utility node: Neither Sink nor Sink Joiner");
- }
-
- if (in2 instanceof SinkPlanNode) {
- sinks.add((SinkPlanNode) in2);
- } else if (in2 instanceof SinkJoinerPlanNode) {
- ((SinkJoinerPlanNode) in2).getDataSinks(sinks);
- } else {
- throw new CompilerException("Illegal child node for a sink joiner utility node: Neither Sink nor Sink Joiner");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
deleted file mode 100644
index 656e67f..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SinkPlanNode.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-
-/**
- * Plan candidate node for data flow sinks.
- */
-public class SinkPlanNode extends SingleInputPlanNode
-{
- /**
- * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that
- * local sorting and range partitioning are handled by the incoming channel already.
- *
- * @param template The template optimizer node that this candidate is created for.
- */
- public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) {
- super(template, nodeName, input, DriverStrategy.NONE);
-
- this.globalProps = input.getGlobalProperties().clone();
- this.localProps = input.getLocalProperties().clone();
- }
-
- public DataSinkNode getSinkNode() {
- if (this.template instanceof DataSinkNode) {
- return (DataSinkNode) this.template;
- } else {
- throw new RuntimeException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
deleted file mode 100644
index 63093dd..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SolutionSetPlanNode.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.Collections;
-import java.util.HashMap;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SolutionSetNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * Plan candidate node for partial solution of a bulk iteration.
- */
-public class SolutionSetPlanNode extends PlanNode {
-
- private static final Costs NO_COSTS = new Costs();
-
- private WorksetIterationPlanNode containingIterationNode;
-
- private final Channel initialInput;
-
- public Object postPassHelper;
-
-
- public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
- GlobalProperties gProps, LocalProperties lProps,
- Channel initialInput)
- {
- super(template, nodeName, DriverStrategy.NONE);
-
- this.globalProps = gProps;
- this.localProps = lProps;
- this.initialInput = initialInput;
-
- // the node incurs no cost
- this.nodeCosts = NO_COSTS;
- this.cumulativeCosts = NO_COSTS;
-
- if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
- }
-
- this.branchPlan.putAll(initialInput.getSource().branchPlan);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public SolutionSetNode getSolutionSetNode() {
- return (SolutionSetNode) this.template;
- }
-
- public WorksetIterationPlanNode getContainingIterationNode() {
- return this.containingIterationNode;
- }
-
- public void setContainingIterationNode(WorksetIterationPlanNode containingIterationNode) {
- this.containingIterationNode = containingIterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- return Collections.<PlanNode>emptyList();
- }
-
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.<Channel>emptyList();
- }
-
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE_AND_DAM;
- }
-
- SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
- if (res == FOUND_SOURCE_AND_DAM || res == FOUND_SOURCE) {
- return FOUND_SOURCE_AND_DAM;
- } else {
- return NOT_FOUND;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
deleted file mode 100644
index 11b7cc9..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/SourcePlanNode.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.Collections;
-
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * Plan candidate node for data flow sources that have no input and no special strategies.
- */
-public class SourcePlanNode extends PlanNode {
-
- private TypeSerializerFactory<?> serializer;
-
- /**
- * Constructs a new source candidate node that uses <i>NONE</i> as its local strategy.
- *
- * @param template The template optimizer node that this candidate is created for.
- */
- public SourcePlanNode(DataSourceNode template, String nodeName) {
- this(template, nodeName, new GlobalProperties(), new LocalProperties());
- }
-
- public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) {
- super(template, nodeName, DriverStrategy.NONE);
-
- this.globalProps = gprops;
- this.localProps = lprops;
- updatePropertiesWithUniqueSets(template.getUniqueFields());
- }
-
- // --------------------------------------------------------------------------------------------
-
- public DataSourceNode getDataSourceNode() {
- return (DataSourceNode) this.template;
- }
-
- /**
- * Gets the serializer from this PlanNode.
- *
- * @return The serializer.
- */
- public TypeSerializerFactory<?> getSerializer() {
- return serializer;
- }
-
- /**
- * Sets the serializer for this PlanNode.
- *
- * @param serializer The serializer to set.
- */
- public void setSerializer(TypeSerializerFactory<?> serializer) {
- this.serializer = serializer;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- return Collections.<PlanNode>emptyList();
- }
-
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.<Channel>emptyList();
- }
-
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- } else {
- return NOT_FOUND;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
deleted file mode 100644
index 880f2e3..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/StreamingPlan.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.flink.runtime.jobgraph.JobGraph;
-
-/**
- * Abstract class representing Flink Streaming plans
- *
- */
-public abstract class StreamingPlan implements FlinkPlan {
-
- public abstract JobGraph getJobGraph();
-
- public abstract String getStreamingPlanAsJSON();
-
- public abstract void dumpStreamingPlanAsJSON(File file) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
deleted file mode 100644
index 95adace..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetIterationPlanNode.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * A node in the execution, representing a workset iteration (delta iteration).
- */
-public class WorksetIterationPlanNode extends DualInputPlanNode implements IterationPlanNode {
-
- private final SolutionSetPlanNode solutionSetPlanNode;
-
- private final WorksetPlanNode worksetPlanNode;
-
- private final PlanNode solutionSetDeltaPlanNode;
-
- private final PlanNode nextWorkSetPlanNode;
-
- private TypeSerializerFactory<?> worksetSerializer;
-
- private TypeSerializerFactory<?> solutionSetSerializer;
-
- private TypeComparatorFactory<?> solutionSetComparator;
-
- private boolean immediateSolutionSetUpdate;
-
- public Object postPassHelper;
-
- private TypeSerializerFactory<?> serializerForIterationChannel;
-
- // --------------------------------------------------------------------------------------------
-
- public WorksetIterationPlanNode(WorksetIterationNode template, String nodeName, Channel initialSolutionSet, Channel initialWorkset,
- SolutionSetPlanNode solutionSetPlanNode, WorksetPlanNode worksetPlanNode,
- PlanNode nextWorkSetPlanNode, PlanNode solutionSetDeltaPlanNode)
- {
- super(template, nodeName, initialSolutionSet, initialWorkset, DriverStrategy.BINARY_NO_OP);
- this.solutionSetPlanNode = solutionSetPlanNode;
- this.worksetPlanNode = worksetPlanNode;
- this.solutionSetDeltaPlanNode = solutionSetDeltaPlanNode;
- this.nextWorkSetPlanNode = nextWorkSetPlanNode;
-
- mergeBranchPlanMaps();
-
- }
-
- // --------------------------------------------------------------------------------------------
-
- public WorksetIterationNode getIterationNode() {
- if (this.template instanceof WorksetIterationNode) {
- return (WorksetIterationNode) this.template;
- } else {
- throw new RuntimeException();
- }
- }
-
- public SolutionSetPlanNode getSolutionSetPlanNode() {
- return this.solutionSetPlanNode;
- }
-
- public WorksetPlanNode getWorksetPlanNode() {
- return this.worksetPlanNode;
- }
-
- public PlanNode getSolutionSetDeltaPlanNode() {
- return this.solutionSetDeltaPlanNode;
- }
-
- public PlanNode getNextWorkSetPlanNode() {
- return this.nextWorkSetPlanNode;
- }
-
- public Channel getInitialSolutionSetInput() {
- return getInput1();
- }
-
- public Channel getInitialWorksetInput() {
- return getInput2();
- }
-
- public void setImmediateSolutionSetUpdate(boolean immediateUpdate) {
- this.immediateSolutionSetUpdate = immediateUpdate;
- }
-
- public boolean isImmediateSolutionSetUpdate() {
- return this.immediateSolutionSetUpdate;
- }
-
- public FieldList getSolutionSetKeyFields() {
- return getIterationNode().getSolutionSetKeyFields();
- }
-
- // --------------------------------------------------------------------------------------------
-
- public TypeSerializerFactory<?> getWorksetSerializer() {
- return worksetSerializer;
- }
-
- public void setWorksetSerializer(TypeSerializerFactory<?> worksetSerializer) {
- this.worksetSerializer = worksetSerializer;
- }
-
- public TypeSerializerFactory<?> getSolutionSetSerializer() {
- return solutionSetSerializer;
- }
-
- public void setSolutionSetSerializer(TypeSerializerFactory<?> solutionSetSerializer) {
- this.solutionSetSerializer = solutionSetSerializer;
- }
-
- public TypeComparatorFactory<?> getSolutionSetComparator() {
- return solutionSetComparator;
- }
-
- public void setSolutionSetComparator(TypeComparatorFactory<?> solutionSetComparator) {
- this.solutionSetComparator = solutionSetComparator;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public void setCosts(Costs nodeCosts) {
- // add the costs from the step function
- nodeCosts.addCosts(this.solutionSetDeltaPlanNode.getCumulativeCostsShare());
- nodeCosts.addCosts(this.nextWorkSetPlanNode.getCumulativeCostsShare());
-
- super.setCosts(nodeCosts);
- }
-
- public int getMemoryConsumerWeight() {
- // solution set index and workset back channel
- return 2;
- }
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
-
- SourceAndDamReport fromOutside = super.hasDamOnPathDownTo(source);
-
- if (fromOutside == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (fromOutside == FOUND_SOURCE) {
- // we always have a dam in the solution set index
- return FOUND_SOURCE_AND_DAM;
- } else {
- SourceAndDamReport fromNextWorkset = nextWorkSetPlanNode.hasDamOnPathDownTo(source);
-
- if (fromNextWorkset == FOUND_SOURCE_AND_DAM){
- return FOUND_SOURCE_AND_DAM;
- } else if (fromNextWorkset == FOUND_SOURCE){
- return FOUND_SOURCE_AND_DAM;
- } else {
- return this.solutionSetDeltaPlanNode.hasDamOnPathDownTo(source);
- }
- }
- }
-
- @Override
- public void acceptForStepFunction(Visitor<PlanNode> visitor) {
- this.solutionSetDeltaPlanNode.accept(visitor);
- this.nextWorkSetPlanNode.accept(visitor);
- }
-
- /**
- * Merging can only take place after the solutionSetDelta and nextWorkset PlanNode has been set,
- * because they can contain also some of the branching nodes.
- */
- @Override
- protected void mergeBranchPlanMaps(Map<OptimizerNode, PlanNode> branchPlan1, Map<OptimizerNode,PlanNode> branchPlan2) {}
-
-
- protected void mergeBranchPlanMaps() {
- Map<OptimizerNode, PlanNode> branchPlan1 = input1.getSource().branchPlan;
- Map<OptimizerNode, PlanNode> branchPlan2 = input2.getSource().branchPlan;
-
- // merge the branchPlan maps according the template's uncloseBranchesStack
- if (this.template.hasUnclosedBranches()) {
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
- }
-
- for (OptimizerNode.UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
- OptimizerNode brancher = uc.getBranchingNode();
- PlanNode selectedCandidate = null;
-
- if (branchPlan1 != null) {
- // predecessor 1 has branching children, see if it got the branch we are looking for
- selectedCandidate = branchPlan1.get(brancher);
- }
-
- if (selectedCandidate == null && branchPlan2 != null) {
- // predecessor 2 has branching children, see if it got the branch we are looking for
- selectedCandidate = branchPlan2.get(brancher);
- }
-
- if(selectedCandidate == null && getSolutionSetDeltaPlanNode() != null && getSolutionSetDeltaPlanNode()
- .branchPlan != null){
- selectedCandidate = getSolutionSetDeltaPlanNode().branchPlan.get(brancher);
- }
-
- if(selectedCandidate == null && getNextWorkSetPlanNode() != null && getNextWorkSetPlanNode()
- .branchPlan != null){
- selectedCandidate = getNextWorkSetPlanNode().branchPlan.get(brancher);
- }
-
- if (selectedCandidate == null) {
- throw new CompilerException(
- "Candidates for a node with open branches are missing information about the selected candidate ");
- }
-
- this.branchPlan.put(brancher, selectedCandidate);
- }
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public TypeSerializerFactory<?> getSerializerForIterationChannel() {
- return serializerForIterationChannel;
- }
-
- public void setSerializerForIterationChannel(TypeSerializerFactory<?> serializerForIterationChannel) {
- this.serializerForIterationChannel = serializerForIterationChannel;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
deleted file mode 100644
index 8d044d6..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plan/WorksetPlanNode.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.optimizer.plan;
-
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
-import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.NOT_FOUND;
-
-import java.util.Collections;
-import java.util.HashMap;
-
-import org.apache.flink.optimizer.costs.Costs;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.WorksetNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.runtime.operators.DamBehavior;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.util.Visitor;
-
-/**
- * Plan candidate node for partial solution of a bulk iteration.
- */
-public class WorksetPlanNode extends PlanNode {
-
- private static final Costs NO_COSTS = new Costs();
-
- private WorksetIterationPlanNode containingIterationNode;
-
- private final Channel initialInput;
-
- public Object postPassHelper;
-
-
- public WorksetPlanNode(WorksetNode template, String nodeName,
- GlobalProperties gProps, LocalProperties lProps,
- Channel initialInput)
- {
- super(template, nodeName, DriverStrategy.NONE);
-
- this.globalProps = gProps;
- this.localProps = lProps;
- this.initialInput = initialInput;
-
- // the node incurs no cost
- this.nodeCosts = NO_COSTS;
- this.cumulativeCosts = NO_COSTS;
-
- if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
- if (this.branchPlan == null) {
- this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
- }
-
- this.branchPlan.putAll(initialInput.getSource().branchPlan);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- public WorksetNode getWorksetNode() {
- return (WorksetNode) this.template;
- }
-
- public WorksetIterationPlanNode getContainingIterationNode() {
- return this.containingIterationNode;
- }
-
- public void setContainingIterationNode(WorksetIterationPlanNode containingIterationNode) {
- this.containingIterationNode = containingIterationNode;
- }
-
- // --------------------------------------------------------------------------------------------
-
-
- @Override
- public void accept(Visitor<PlanNode> visitor) {
- if (visitor.preVisit(this)) {
- visitor.postVisit(this);
- }
- }
-
-
- @Override
- public Iterable<PlanNode> getPredecessors() {
- return Collections.<PlanNode>emptyList();
- }
-
-
- @Override
- public Iterable<Channel> getInputs() {
- return Collections.<Channel>emptyList();
- }
-
-
- @Override
- public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
- if (source == this) {
- return FOUND_SOURCE;
- }
- SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source);
- if (res == FOUND_SOURCE_AND_DAM) {
- return FOUND_SOURCE_AND_DAM;
- }
- else if (res == FOUND_SOURCE) {
- return (this.initialInput.getLocalStrategy().dams() ||
- this.initialInput.getTempMode().breaksPipeline() ||
- getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ?
- FOUND_SOURCE_AND_DAM : FOUND_SOURCE;
- }
- else {
- return NOT_FOUND;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
deleted file mode 100644
index 3f8cb46..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableConnection.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plandump;
-
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-
-
-/**
- *
- */
-public interface DumpableConnection<T extends DumpableNode<T>> {
-
- public DumpableNode<T> getSource();
-
- public ShipStrategyType getShipStrategy();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
deleted file mode 100644
index 1bc0f0c..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/DumpableNode.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plandump;
-
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.plan.PlanNode;
-
-/**
- *
- */
-public interface DumpableNode<T extends DumpableNode<T>> {
-
- /**
- * Gets an iterator over the predecessors.
- *
- * @return An iterator over the predecessors.
- */
- Iterable<T> getPredecessors();
-
- Iterable<DumpableConnection<T>> getDumpableInputs();
-
- OptimizerNode getOptimizerNode();
-
- PlanNode getPlanNode();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java b/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
deleted file mode 100644
index 6f918c0..0000000
--- a/flink-compiler/src/main/java/org/apache/flink/optimizer/plandump/PlanJSONDumpGenerator.java
+++ /dev/null
@@ -1,657 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.plandump;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringEscapeUtils;
-import org.apache.flink.api.common.operators.CompilerHints;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.BinaryUnionNode;
-import org.apache.flink.optimizer.dag.BulkIterationNode;
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.DataSourceNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.DagConnection;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.dag.WorksetIterationNode;
-import org.apache.flink.optimizer.dataproperties.GlobalProperties;
-import org.apache.flink.optimizer.dataproperties.LocalProperties;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.optimizer.util.Utils;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.util.StringUtils;
-
-/**
- *
- */
-public class PlanJSONDumpGenerator {
-
- private Map<DumpableNode<?>, Integer> nodeIds; // resolves pact nodes to ids
-
- private int nodeCnt;
-
- private boolean encodeForHTML;
-
- // --------------------------------------------------------------------------------------------
-
- public void setEncodeForHTML(boolean encodeForHTML) {
- this.encodeForHTML = encodeForHTML;
- }
-
- public boolean isEncodeForHTML() {
- return encodeForHTML;
- }
-
-
- public void dumpPactPlanAsJSON(List<DataSinkNode> nodes, PrintWriter writer) {
- @SuppressWarnings("unchecked")
- List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
- compilePlanToJSON(n, writer);
- }
-
- public String getPactPlanAsJSON(List<DataSinkNode> nodes) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- dumpPactPlanAsJSON(nodes, pw);
- return sw.toString();
- }
-
- public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, File toFile) throws IOException {
- PrintWriter pw = null;
- try {
- pw = new PrintWriter(new FileOutputStream(toFile), false);
- dumpOptimizerPlanAsJSON(plan, pw);
- pw.flush();
- } finally {
- if (pw != null) {
- pw.close();
- }
- }
- }
-
- public String getOptimizerPlanAsJSON(OptimizedPlan plan) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- dumpOptimizerPlanAsJSON(plan, pw);
- pw.close();
- return sw.toString();
- }
-
- public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) {
- Collection<SinkPlanNode> sinks = plan.getDataSinks();
- if (sinks instanceof List) {
- dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer);
- } else {
- List<SinkPlanNode> n = new ArrayList<SinkPlanNode>();
- n.addAll(sinks);
- dumpOptimizerPlanAsJSON(n, writer);
- }
- }
-
- public void dumpOptimizerPlanAsJSON(List<SinkPlanNode> nodes, PrintWriter writer) {
- @SuppressWarnings("unchecked")
- List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
- compilePlanToJSON(n, writer);
- }
-
- // --------------------------------------------------------------------------------------------
-
- private void compilePlanToJSON(List<DumpableNode<?>> nodes, PrintWriter writer) {
- // initialization to assign node ids
- this.nodeIds = new HashMap<DumpableNode<?>, Integer>();
- this.nodeCnt = 0;
-
- // JSON header
- writer.print("{\n\t\"nodes\": [\n\n");
-
- // Generate JSON for plan
- for (int i = 0; i < nodes.size(); i++) {
- visit(nodes.get(i), writer, i == 0);
- }
-
- // JSON Footer
- writer.println("\n\t]\n}");
- }
-
- private boolean visit(DumpableNode<?> node, PrintWriter writer, boolean first) {
- // check for duplicate traversal
- if (this.nodeIds.containsKey(node)) {
- return false;
- }
-
- // assign an id first
- this.nodeIds.put(node, this.nodeCnt++);
-
- // then recurse
- for (DumpableNode<?> child : node.getPredecessors()) {
- //This is important, because when the node was already in the graph it is not allowed
- //to set first to false!
- if (visit(child, writer, first)) {
- first = false;
- };
- }
-
- // check if this node should be skipped from the dump
- final OptimizerNode n = node.getOptimizerNode();
-
- // ------------------ dump after the ascend ---------------------
- // start a new node and output node id
- if (!first) {
- writer.print(",\n");
- }
- // open the node
- writer.print("\t{\n");
-
- // recurse, it is is an iteration node
- if (node instanceof BulkIterationNode || node instanceof BulkIterationPlanNode) {
-
- DumpableNode<?> innerChild = node instanceof BulkIterationNode ?
- ((BulkIterationNode) node).getNextPartialSolution() :
- ((BulkIterationPlanNode) node).getRootOfStepFunction();
-
- DumpableNode<?> begin = node instanceof BulkIterationNode ?
- ((BulkIterationNode) node).getPartialSolution() :
- ((BulkIterationPlanNode) node).getPartialSolutionPlanNode();
-
- writer.print("\t\t\"step_function\": [\n");
-
- visit(innerChild, writer, true);
-
- writer.print("\n\t\t],\n");
- writer.print("\t\t\"partial_solution\": " + this.nodeIds.get(begin) + ",\n");
- writer.print("\t\t\"next_partial_solution\": " + this.nodeIds.get(innerChild) + ",\n");
- } else if (node instanceof WorksetIterationNode || node instanceof WorksetIterationPlanNode) {
-
- DumpableNode<?> worksetRoot = node instanceof WorksetIterationNode ?
- ((WorksetIterationNode) node).getNextWorkset() :
- ((WorksetIterationPlanNode) node).getNextWorkSetPlanNode();
- DumpableNode<?> solutionDelta = node instanceof WorksetIterationNode ?
- ((WorksetIterationNode) node).getSolutionSetDelta() :
- ((WorksetIterationPlanNode) node).getSolutionSetDeltaPlanNode();
-
- DumpableNode<?> workset = node instanceof WorksetIterationNode ?
- ((WorksetIterationNode) node).getWorksetNode() :
- ((WorksetIterationPlanNode) node).getWorksetPlanNode();
- DumpableNode<?> solutionSet = node instanceof WorksetIterationNode ?
- ((WorksetIterationNode) node).getSolutionSetNode() :
- ((WorksetIterationPlanNode) node).getSolutionSetPlanNode();
-
- writer.print("\t\t\"step_function\": [\n");
-
- visit(worksetRoot, writer, true);
- visit(solutionDelta, writer, false);
-
- writer.print("\n\t\t],\n");
- writer.print("\t\t\"workset\": " + this.nodeIds.get(workset) + ",\n");
- writer.print("\t\t\"solution_set\": " + this.nodeIds.get(solutionSet) + ",\n");
- writer.print("\t\t\"next_workset\": " + this.nodeIds.get(worksetRoot) + ",\n");
- writer.print("\t\t\"solution_delta\": " + this.nodeIds.get(solutionDelta) + ",\n");
- }
-
- // print the id
- writer.print("\t\t\"id\": " + this.nodeIds.get(node));
-
-
- final String type;
- String contents;
- if (n instanceof DataSinkNode) {
- type = "sink";
- contents = n.getOperator().toString();
- } else if (n instanceof DataSourceNode) {
- type = "source";
- contents = n.getOperator().toString();
- }
- else if (n instanceof BulkIterationNode) {
- type = "bulk_iteration";
- contents = n.getOperator().getName();
- }
- else if (n instanceof WorksetIterationNode) {
- type = "workset_iteration";
- contents = n.getOperator().getName();
- }
- else if (n instanceof BinaryUnionNode) {
- type = "pact";
- contents = "";
- }
- else {
- type = "pact";
- contents = n.getOperator().getName();
- }
-
- contents = StringUtils.showControlCharacters(contents);
- if (encodeForHTML) {
- contents = StringEscapeUtils.escapeHtml4(contents);
- contents = contents.replace("\\", "\");
- }
-
-
- String name = n.getName();
- if (name.equals("Reduce") && (node instanceof SingleInputPlanNode) &&
- ((SingleInputPlanNode) node).getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE) {
- name = "Combine";
- }
-
- // output the type identifier
- writer.print(",\n\t\t\"type\": \"" + type + "\"");
-
- // output node name
- writer.print(",\n\t\t\"pact\": \"" + name + "\"");
-
- // output node contents
- writer.print(",\n\t\t\"contents\": \"" + contents + "\"");
-
- // degree of parallelism
- writer.print(",\n\t\t\"parallelism\": \""
- + (n.getParallelism() >= 1 ? n.getParallelism() : "default") + "\"");
-
- // output node predecessors
- Iterator<? extends DumpableConnection<?>> inConns = node.getDumpableInputs().iterator();
- String child1name = "", child2name = "";
-
- if (inConns != null && inConns.hasNext()) {
- // start predecessor list
- writer.print(",\n\t\t\"predecessors\": [");
- int inputNum = 0;
-
- while (inConns.hasNext()) {
- final DumpableConnection<?> inConn = inConns.next();
- final DumpableNode<?> source = inConn.getSource();
- writer.print(inputNum == 0 ? "\n" : ",\n");
- if (inputNum == 0) {
- child1name += child1name.length() > 0 ? ", " : "";
- child1name += source.getOptimizerNode().getOperator().getName();
- } else if (inputNum == 1) {
- child2name += child2name.length() > 0 ? ", " : "";
- child2name = source.getOptimizerNode().getOperator().getName();
- }
-
- // output predecessor id
- writer.print("\t\t\t{\"id\": " + this.nodeIds.get(source));
-
- // output connection side
- if (inConns.hasNext() || inputNum > 0) {
- writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
- }
- // output shipping strategy and channel type
- final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
- final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
- ((DagConnection) inConn).getShipStrategy();
-
- String shipStrategy = null;
- if (shipType != null) {
- switch (shipType) {
- case NONE:
- // nothing
- break;
- case FORWARD:
- shipStrategy = "Forward";
- break;
- case BROADCAST:
- shipStrategy = "Broadcast";
- break;
- case PARTITION_HASH:
- shipStrategy = "Hash Partition";
- break;
- case PARTITION_RANGE:
- shipStrategy = "Range Partition";
- break;
- case PARTITION_RANDOM:
- shipStrategy = "Redistribute";
- break;
- case PARTITION_FORCED_REBALANCE:
- shipStrategy = "Rebalance";
- break;
- case PARTITION_CUSTOM:
- shipStrategy = "Custom Partition";
- break;
- default:
- throw new CompilerException("Unknown ship strategy '" + inConn.getShipStrategy().name()
- + "' in JSON generator.");
- }
- }
-
- if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
- shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ?
- channel.getShipStrategyKeys().toString() :
- Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
- }
-
- if (shipStrategy != null) {
- writer.print(", \"ship_strategy\": \"" + shipStrategy + "\"");
- }
-
- if (channel != null) {
- String localStrategy = null;
- switch (channel.getLocalStrategy()) {
- case NONE:
- break;
- case SORT:
- localStrategy = "Sort";
- break;
- case COMBININGSORT:
- localStrategy = "Sort (combining)";
- break;
- default:
- throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name());
- }
-
- if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
- localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ?
- channel.getLocalStrategyKeys().toString() :
- Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
- }
-
- if (localStrategy != null) {
- writer.print(", \"local_strategy\": \"" + localStrategy + "\"");
- }
-
- if (channel != null && channel.getTempMode() != TempMode.NONE) {
- String tempMode = channel.getTempMode().toString();
- writer.print(", \"temp_mode\": \"" + tempMode + "\"");
- }
- }
-
- writer.print('}');
- inputNum++;
- }
- // finish predecessors
- writer.print("\n\t\t]");
- }
-
- //---------------------------------------------------------------------------------------
- // the part below here is relevant only to plan nodes with concrete strategies, etc
- //---------------------------------------------------------------------------------------
-
- final PlanNode p = node.getPlanNode();
- if (p == null) {
- // finish node
- writer.print("\n\t}");
- return true;
- }
- // local strategy
- String locString = null;
- if (p.getDriverStrategy() != null) {
- switch (p.getDriverStrategy()) {
- case NONE:
- case BINARY_NO_OP:
- break;
-
- case UNARY_NO_OP:
- locString = "No-Op";
- break;
-
- case COLLECTOR_MAP:
- case MAP:
- locString = "Map";
- break;
-
- case FLAT_MAP:
- locString = "FlatMap";
- break;
-
- case MAP_PARTITION:
- locString = "Map Partition";
- break;
-
- case ALL_REDUCE:
- locString = "Reduce All";
- break;
-
- case ALL_GROUP_REDUCE:
- case ALL_GROUP_REDUCE_COMBINE:
- locString = "Group Reduce All";
- break;
-
- case SORTED_REDUCE:
- locString = "Sorted Reduce";
- break;
-
- case SORTED_PARTIAL_REDUCE:
- locString = "Sorted Combine/Reduce";
- break;
-
- case SORTED_GROUP_REDUCE:
- locString = "Sorted Group Reduce";
- break;
-
- case SORTED_GROUP_COMBINE:
- locString = "Sorted Combine";
- break;
-
- case HYBRIDHASH_BUILD_FIRST:
- locString = "Hybrid Hash (build: " + child1name + ")";
- break;
- case HYBRIDHASH_BUILD_SECOND:
- locString = "Hybrid Hash (build: " + child2name + ")";
- break;
-
- case HYBRIDHASH_BUILD_FIRST_CACHED:
- locString = "Hybrid Hash (CACHED) (build: " + child1name + ")";
- break;
- case HYBRIDHASH_BUILD_SECOND_CACHED:
- locString = "Hybrid Hash (CACHED) (build: " + child2name + ")";
- break;
-
- case NESTEDLOOP_BLOCKED_OUTER_FIRST:
- locString = "Nested Loops (Blocked Outer: " + child1name + ")";
- break;
- case NESTEDLOOP_BLOCKED_OUTER_SECOND:
- locString = "Nested Loops (Blocked Outer: " + child2name + ")";
- break;
- case NESTEDLOOP_STREAMED_OUTER_FIRST:
- locString = "Nested Loops (Streamed Outer: " + child1name + ")";
- break;
- case NESTEDLOOP_STREAMED_OUTER_SECOND:
- locString = "Nested Loops (Streamed Outer: " + child2name + ")";
- break;
-
- case MERGE:
- locString = "Merge";
- break;
-
- case CO_GROUP:
- locString = "Co-Group";
- break;
-
- default:
- locString = p.getDriverStrategy().name();
- break;
- }
-
- if (locString != null) {
- writer.print(",\n\t\t\"driver_strategy\": \"");
- writer.print(locString);
- writer.print("\"");
- }
- }
-
- {
- // output node global properties
- final GlobalProperties gp = p.getGlobalProperties();
-
- writer.print(",\n\t\t\"global_properties\": [\n");
-
- addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
- if (gp.getPartitioningFields() != null) {
- addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
- }
- if (gp.getPartitioningOrdering() != null) {
- addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false);
- }
- else {
- addProperty(writer, "Partitioning Order", "(none)", false);
- }
- if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
- addProperty(writer, "Uniqueness", "not unique", false);
- }
- else {
- addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);
- }
-
- writer.print("\n\t\t]");
- }
-
- {
- // output node local properties
- LocalProperties lp = p.getLocalProperties();
-
- writer.print(",\n\t\t\"local_properties\": [\n");
-
- if (lp.getOrdering() != null) {
- addProperty(writer, "Order", lp.getOrdering().toString(), true);
- }
- else {
- addProperty(writer, "Order", "(none)", true);
- }
- if (lp.getGroupedFields() != null && lp.getGroupedFields().size() > 0) {
- addProperty(writer, "Grouped on", lp.getGroupedFields().toString(), false);
- } else {
- addProperty(writer, "Grouping", "not grouped", false);
- }
- if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
- addProperty(writer, "Uniqueness", "not unique", false);
- }
- else {
- addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);
- }
-
- writer.print("\n\t\t]");
- }
-
- // output node size estimates
- writer.print(",\n\t\t\"estimates\": [\n");
-
- addProperty(writer, "Est. Output Size", n.getEstimatedOutputSize() == -1 ? "(unknown)"
- : formatNumber(n.getEstimatedOutputSize(), "B"), true);
- addProperty(writer, "Est. Cardinality", n.getEstimatedNumRecords() == -1 ? "(unknown)"
- : formatNumber(n.getEstimatedNumRecords()), false);
-
- writer.print("\t\t]");
-
- // output node cost
- if (p.getNodeCosts() != null) {
- writer.print(",\n\t\t\"costs\": [\n");
-
- addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)"
- : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true);
- addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)"
- : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false);
- addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)"
- : formatNumber(p.getNodeCosts().getCpuCost(), ""), false);
-
- addProperty(writer, "Cumulative Network",
- p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p
- .getCumulativeCosts().getNetworkCost(), "B"), false);
- addProperty(writer, "Cumulative Disk I/O",
- p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p
- .getCumulativeCosts().getDiskCost(), "B"), false);
- addProperty(writer, "Cumulative CPU",
- p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
- .getCumulativeCosts().getCpuCost(), ""), false);
-
- writer.print("\n\t\t]");
- }
-
- // output the node compiler hints
- if (n.getOperator().getCompilerHints() != null) {
- CompilerHints hints = n.getOperator().getCompilerHints();
- CompilerHints defaults = new CompilerHints();
-
- String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize());
- String card = hints.getOutputCardinality() == defaults.getOutputCardinality() ? "(none)" : String.valueOf(hints.getOutputCardinality());
- String width = hints.getAvgOutputRecordSize() == defaults.getAvgOutputRecordSize() ? "(none)" : String.valueOf(hints.getAvgOutputRecordSize());
- String filter = hints.getFilterFactor() == defaults.getFilterFactor() ? "(none)" : String.valueOf(hints.getFilterFactor());
-
- writer.print(",\n\t\t\"compiler_hints\": [\n");
-
- addProperty(writer, "Output Size (bytes)", size, true);
- addProperty(writer, "Output Cardinality", card, false);
- addProperty(writer, "Avg. Output Record Size (bytes)", width, false);
- addProperty(writer, "Filter Factor", filter, false);
-
- writer.print("\t\t]");
- }
-
- // finish node
- writer.print("\n\t}");
- return true;
- }
-
- private void addProperty(PrintWriter writer, String name, String value, boolean first) {
- if (!first) {
- writer.print(",\n");
- }
- writer.print("\t\t\t{ \"name\": \"");
- writer.print(name);
- writer.print("\", \"value\": \"");
- writer.print(value);
- writer.print("\" }");
- }
-
- public static final String formatNumber(double number) {
- return formatNumber(number, "");
- }
-
- public static final String formatNumber(double number, String suffix) {
- if (number <= 0.0) {
- return String.valueOf(number);
- }
-
- int power = (int) Math.ceil(Math.log10(number));
-
- int group = (power - 1) / 3;
- if (group >= SIZE_SUFFIXES.length) {
- group = SIZE_SUFFIXES.length - 1;
- } else if (group < 0) {
- group = 0;
- }
-
- // truncate fractional part
- int beforeDecimal = power - group * 3;
- if (power > beforeDecimal) {
- for (int i = power - beforeDecimal; i > 0; i--) {
- number /= 10;
- }
- }
-
- return group > 0 ? String.format(Locale.US, "%.2f %s", number, SIZE_SUFFIXES[group]) :
- String.format(Locale.US, "%.2f", number);
- }
-
- private static final char[] SIZE_SUFFIXES = { 0, 'K', 'M', 'G', 'T' };
-}
[16/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
new file mode 100644
index 0000000..e85f289
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetIterationNode.java
@@ -0,0 +1,589 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.SemanticProperties.EmptySemanticProperties;
+import org.apache.flink.api.common.operators.base.DeltaIterationBase;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.typeinfo.NothingTypeInfo;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.traversals.InterestingPropertyVisitor;
+import org.apache.flink.optimizer.costs.CostEstimator;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.InterestingProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.operators.OperatorDescriptorDual;
+import org.apache.flink.optimizer.operators.SolutionSetDeltaOperator;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.NamedChannel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
+import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+import org.apache.flink.optimizer.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
+import org.apache.flink.optimizer.util.NoOpBinaryUdfOp;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.apache.flink.types.Nothing;
+import org.apache.flink.util.Visitor;
+
+/**
+ * A node in the optimizer's program representation for a workset iteration.
+ */
+public class WorksetIterationNode extends TwoInputNode implements IterationNode {
+
+ private static final int DEFAULT_COST_WEIGHT = 20;
+
+
+ private final FieldList solutionSetKeyFields;
+
+ private final GlobalProperties partitionedProperties;
+
+ private final List<OperatorDescriptorDual> dataProperties;
+
+ private SolutionSetNode solutionSetNode;
+
+ private WorksetNode worksetNode;
+
+ private OptimizerNode solutionSetDelta;
+
+ private OptimizerNode nextWorkset;
+
+ private DagConnection solutionSetDeltaRootConnection;
+
+ private DagConnection nextWorksetRootConnection;
+
+ private SingleRootJoiner singleRoot;
+
+ private boolean solutionDeltaImmediatelyAfterSolutionJoin;
+
+ private final int costWeight;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new node with a single input for the optimizer plan.
+ *
+ * @param iteration The iteration operator that the node represents.
+ */
+ public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
+ super(iteration);
+
+ final int[] ssKeys = iteration.getSolutionSetKeyFields();
+ if (ssKeys == null || ssKeys.length == 0) {
+ throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
+ }
+ this.solutionSetKeyFields = new FieldList(ssKeys);
+ this.partitionedProperties = new GlobalProperties();
+ this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
+
+ int weight = iteration.getMaximumNumberOfIterations() > 0 ?
+ iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
+
+ if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
+ weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
+ }
+ this.costWeight = weight;
+
+ this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public DeltaIterationBase<?, ?> getIterationContract() {
+ return (DeltaIterationBase<?, ?>) getOperator();
+ }
+
+ public SolutionSetNode getSolutionSetNode() {
+ return this.solutionSetNode;
+ }
+
+ public WorksetNode getWorksetNode() {
+ return this.worksetNode;
+ }
+
+ public OptimizerNode getNextWorkset() {
+ return this.nextWorkset;
+ }
+
+ public OptimizerNode getSolutionSetDelta() {
+ return this.solutionSetDelta;
+ }
+
+ public void setPartialSolution(SolutionSetNode solutionSetNode, WorksetNode worksetNode) {
+ if (this.solutionSetNode != null || this.worksetNode != null) {
+ throw new IllegalStateException("Error: Initializing WorksetIterationNode multiple times.");
+ }
+ this.solutionSetNode = solutionSetNode;
+ this.worksetNode = worksetNode;
+ }
+
+ public void setNextPartialSolution(OptimizerNode solutionSetDelta, OptimizerNode nextWorkset,
+ ExecutionMode executionMode) {
+
+ // check whether the next partial solution is itself the join with
+ // the partial solution (so we can potentially do direct updates)
+ if (solutionSetDelta instanceof TwoInputNode) {
+ TwoInputNode solutionDeltaTwoInput = (TwoInputNode) solutionSetDelta;
+ if (solutionDeltaTwoInput.getFirstPredecessorNode() == this.solutionSetNode ||
+ solutionDeltaTwoInput.getSecondPredecessorNode() == this.solutionSetNode)
+ {
+ this.solutionDeltaImmediatelyAfterSolutionJoin = true;
+ }
+ }
+
+ // there needs to be at least one node in the workset path, so
+ // if the next workset is equal to the workset, we need to inject a no-op node
+ if (nextWorkset == worksetNode || nextWorkset instanceof BinaryUnionNode) {
+ NoOpNode noop = new NoOpNode();
+ noop.setDegreeOfParallelism(getParallelism());
+
+ DagConnection noOpConn = new DagConnection(nextWorkset, noop, executionMode);
+ noop.setIncomingConnection(noOpConn);
+ nextWorkset.addOutgoingConnection(noOpConn);
+
+ nextWorkset = noop;
+ }
+
+ // attach an extra node to the solution set delta for the cases where we need to repartition
+ UnaryOperatorNode solutionSetDeltaUpdateAux = new UnaryOperatorNode("Solution-Set Delta", getSolutionSetKeyFields(),
+ new SolutionSetDeltaOperator(getSolutionSetKeyFields()));
+ solutionSetDeltaUpdateAux.setDegreeOfParallelism(getParallelism());
+
+ DagConnection conn = new DagConnection(solutionSetDelta, solutionSetDeltaUpdateAux, executionMode);
+ solutionSetDeltaUpdateAux.setIncomingConnection(conn);
+ solutionSetDelta.addOutgoingConnection(conn);
+
+ this.solutionSetDelta = solutionSetDeltaUpdateAux;
+ this.nextWorkset = nextWorkset;
+
+ this.singleRoot = new SingleRootJoiner();
+ this.solutionSetDeltaRootConnection = new DagConnection(solutionSetDeltaUpdateAux,
+ this.singleRoot, executionMode);
+
+ this.nextWorksetRootConnection = new DagConnection(nextWorkset, this.singleRoot, executionMode);
+ this.singleRoot.setInputs(this.solutionSetDeltaRootConnection, this.nextWorksetRootConnection);
+
+ solutionSetDeltaUpdateAux.addOutgoingConnection(this.solutionSetDeltaRootConnection);
+ nextWorkset.addOutgoingConnection(this.nextWorksetRootConnection);
+ }
+
+ public int getCostWeight() {
+ return this.costWeight;
+ }
+
+ public TwoInputNode getSingleRootOfStepFunction() {
+ return this.singleRoot;
+ }
+
+ public FieldList getSolutionSetKeyFields() {
+ return this.solutionSetKeyFields;
+ }
+
+ public OptimizerNode getInitialSolutionSetPredecessorNode() {
+ return getFirstPredecessorNode();
+ }
+
+ public OptimizerNode getInitialWorksetPredecessorNode() {
+ return getSecondPredecessorNode();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public String getName() {
+ return "Workset Iteration";
+ }
+
+ @Override
+ public SemanticProperties getSemanticProperties() {
+ return new EmptySemanticProperties();
+ }
+
+ protected void readStubAnnotations() {}
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ this.estimatedOutputSize = getFirstPredecessorNode().getEstimatedOutputSize();
+ this.estimatedNumRecords = getFirstPredecessorNode().getEstimatedNumRecords();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Properties and Optimization
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return this.dataProperties;
+ }
+
+ @Override
+ public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
+ // our own solution (the solution set) is always partitioned and this cannot be adjusted
+ // depending on what the successor to the workset iteration requests. for that reason,
+ // we ignore incoming interesting properties.
+
+ // in addition, we need to make 2 interesting property passes, because the root of the step function
+ // that computes the next workset needs the interesting properties as generated by the
+ // workset source of the step function. the second pass concerns only the workset path.
+ // as initial interesting properties, we have the trivial ones for the step function,
+ // and partitioned on the solution set key for the solution set delta
+
+ RequestedGlobalProperties partitionedProperties = new RequestedGlobalProperties();
+ partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
+ InterestingProperties partitionedIP = new InterestingProperties();
+ partitionedIP.addGlobalProperties(partitionedProperties);
+ partitionedIP.addLocalProperties(new RequestedLocalProperties());
+
+ this.nextWorksetRootConnection.setInterestingProperties(new InterestingProperties());
+ this.solutionSetDeltaRootConnection.setInterestingProperties(partitionedIP.clone());
+
+ InterestingPropertyVisitor ipv = new InterestingPropertyVisitor(estimator);
+ this.nextWorkset.accept(ipv);
+ this.solutionSetDelta.accept(ipv);
+
+ // take the interesting properties of the partial solution and add them to the root interesting properties
+ InterestingProperties worksetIntProps = this.worksetNode.getInterestingProperties();
+ InterestingProperties intProps = new InterestingProperties();
+ intProps.getGlobalProperties().addAll(worksetIntProps.getGlobalProperties());
+ intProps.getLocalProperties().addAll(worksetIntProps.getLocalProperties());
+
+ // clear all interesting properties to prepare the second traversal
+ this.nextWorksetRootConnection.clearInterestingProperties();
+ this.nextWorkset.accept(InterestingPropertiesClearer.INSTANCE);
+
+ // 2nd pass
+ this.nextWorksetRootConnection.setInterestingProperties(intProps);
+ this.nextWorkset.accept(ipv);
+
+ // now add the interesting properties of the workset to the workset input
+ final InterestingProperties inProps = this.worksetNode.getInterestingProperties().clone();
+ inProps.addGlobalProperties(new RequestedGlobalProperties());
+ inProps.addLocalProperties(new RequestedLocalProperties());
+ this.input2.setInterestingProperties(inProps);
+
+ // the partial solution must be hash partitioned, so it has only that as interesting properties
+ this.input1.setInterestingProperties(partitionedIP);
+ }
+
+ @Override
+ public void clearInterestingProperties() {
+ super.clearInterestingProperties();
+
+ this.nextWorksetRootConnection.clearInterestingProperties();
+ this.solutionSetDeltaRootConnection.clearInterestingProperties();
+
+ this.nextWorkset.accept(InterestingPropertiesClearer.INSTANCE);
+ this.solutionSetDelta.accept(InterestingPropertiesClearer.INSTANCE);
+ }
+
+ @Override
+ protected void instantiate(OperatorDescriptorDual operator, Channel solutionSetIn, Channel worksetIn,
+ List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
+ RequestedGlobalProperties globPropsReqSolutionSet, RequestedGlobalProperties globPropsReqWorkset,
+ RequestedLocalProperties locPropsReqSolutionSet, RequestedLocalProperties locPropsReqWorkset)
+ {
+ // check for pipeline breaking using hash join with build on the solution set side
+ placePipelineBreakersIfNecessary(DriverStrategy.HYBRIDHASH_BUILD_FIRST, solutionSetIn, worksetIn);
+
+ // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
+ // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
+ // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
+ // we have a fitting candidate for the step function (often, work is pushed out of the step function).
+ // Among the candidates of the step function, we keep only those that meet the requested properties of the
+ // current candidate initial partial solution. That makes sure these properties exist at the beginning of
+ // every iteration.
+
+ // 1) Because we enumerate multiple times, we may need to clean the cached plans
+ // before starting another enumeration
+ this.nextWorkset.accept(PlanCacheCleaner.INSTANCE);
+ this.solutionSetDelta.accept(PlanCacheCleaner.INSTANCE);
+
+ // 2) Give the partial solution the properties of the current candidate for the initial partial solution
+ // This concerns currently only the workset.
+ this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn);
+ this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn);
+
+ final SolutionSetPlanNode sspn = this.solutionSetNode.getCurrentSolutionSetPlanNode();
+ final WorksetPlanNode wspn = this.worksetNode.getCurrentWorksetPlanNode();
+
+ // 3) Get the alternative plans
+ List<PlanNode> solutionSetDeltaCandidates = this.solutionSetDelta.getAlternativePlans(estimator);
+ List<PlanNode> worksetCandidates = this.nextWorkset.getAlternativePlans(estimator);
+
+ // 4) Throw away all that are not compatible with the properties currently requested to the
+ // initial partial solution
+
+ // Make sure that the workset candidates fulfill the input requirements
+ {
+ List<PlanNode> newCandidates = new ArrayList<PlanNode>();
+
+ for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
+ PlanNode candidate = planDeleter.next();
+
+ GlobalProperties atEndGlobal = candidate.getGlobalProperties();
+ LocalProperties atEndLocal = candidate.getLocalProperties();
+
+ FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn,
+ atEndGlobal, atEndLocal);
+
+ if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
+ ; // depends only through broadcast variable on the workset solution
+ }
+ else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
+ // attach a no-op node through which we create the properties of the original input
+ Channel toNoOp = new Channel(candidate);
+ globPropsReqWorkset.parameterizeChannel(toNoOp, false,
+ nextWorksetRootConnection.getDataExchangeMode(), false);
+ locPropsReqWorkset.parameterizeChannel(toNoOp);
+
+ UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties",
+ FieldList.EMPTY_LIST);
+
+ rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getParallelism());
+
+ SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(
+ rebuildWorksetPropertiesNode, "Rebuild Workset Properties",
+ toNoOp, DriverStrategy.UNARY_NO_OP);
+ rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(),
+ toNoOp.getLocalProperties());
+ estimator.costOperator(rebuildWorksetPropertiesPlanNode);
+
+ GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
+ LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
+
+ if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
+ FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(
+ wspn, atEndGlobalModified, atEndLocalModified);
+ if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
+ newCandidates.add(rebuildWorksetPropertiesPlanNode);
+ }
+ }
+
+ // remove the original operator and add the modified candidate
+ planDeleter.remove();
+
+ }
+ }
+
+ worksetCandidates.addAll(newCandidates);
+ }
+
+ if (worksetCandidates.isEmpty()) {
+ return;
+ }
+
+ // sanity check the solution set delta
+ for (PlanNode solutionSetDeltaCandidate : solutionSetDeltaCandidates) {
+ SingleInputPlanNode candidate = (SingleInputPlanNode) solutionSetDeltaCandidate;
+ GlobalProperties gp = candidate.getGlobalProperties();
+
+ if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null ||
+ !gp.getPartitioningFields().equals(this.solutionSetKeyFields)) {
+ throw new CompilerException("Bug: The solution set delta is not partitioned.");
+ }
+ }
+
+ // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
+
+ final GlobalProperties gp = new GlobalProperties();
+ gp.setHashPartitioned(this.solutionSetKeyFields);
+ gp.addUniqueFieldCombination(this.solutionSetKeyFields);
+
+ LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
+
+ // take all combinations of solution set delta and workset plans
+ for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
+ for (PlanNode worksetCandidate : worksetCandidates) {
+ // check whether they have the same operator at their latest branching point
+ if (this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
+
+ SingleInputPlanNode siSolutionDeltaCandidate = (SingleInputPlanNode) solutionSetCandidate;
+ boolean immediateDeltaUpdate;
+
+ // check whether we need a dedicated solution set delta operator, or whether we can update on the fly
+ if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD &&
+ this.solutionDeltaImmediatelyAfterSolutionJoin)
+ {
+ // we do not need this extra node. we can make the predecessor the delta
+ // sanity check the node and connection
+ if (siSolutionDeltaCandidate.getDriverStrategy() != DriverStrategy.UNARY_NO_OP ||
+ siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE)
+ {
+ throw new CompilerException("Invalid Solution set delta node.");
+ }
+
+ solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource();
+ immediateDeltaUpdate = true;
+ } else {
+ // was not partitioned, we need to keep this node.
+ // mark that we materialize the input
+ siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
+ immediateDeltaUpdate = false;
+ }
+
+ WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this,
+ "WorksetIteration ("+this.getOperator().getName()+")", solutionSetIn,
+ worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
+ wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
+ wsNode.initProperties(gp, lp);
+ target.add(wsNode);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ // IMPORTANT: First compute closed branches from the two inputs
+ // we need to do this because the runtime iteration head effectively joins
+ addClosedBranches(getFirstPredecessorNode().closedBranchingNodes);
+ addClosedBranches(getSecondPredecessorNode().closedBranchingNodes);
+
+ List<UnclosedBranchDescriptor> result1 = getFirstPredecessorNode().getBranchesForParent(getFirstIncomingConnection());
+ List<UnclosedBranchDescriptor> result2 = getSecondPredecessorNode().getBranchesForParent(getSecondIncomingConnection());
+
+ ArrayList<UnclosedBranchDescriptor> inputsMerged1 = new ArrayList<UnclosedBranchDescriptor>();
+ mergeLists(result1, result2, inputsMerged1, true); // this method also sets which branches are joined here (in the head)
+
+ addClosedBranches(getSingleRootOfStepFunction().closedBranchingNodes);
+
+ ArrayList<UnclosedBranchDescriptor> inputsMerged2 = new ArrayList<UnclosedBranchDescriptor>();
+ List<UnclosedBranchDescriptor> result3 = getSingleRootOfStepFunction().openBranches;
+ mergeLists(inputsMerged1, result3, inputsMerged2, true);
+
+ // handle the data flow branching for the broadcast inputs
+ List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(inputsMerged2);
+
+ this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Iteration Specific Traversals
+ // --------------------------------------------------------------------------------------------
+
+ public void acceptForStepFunction(Visitor<OptimizerNode> visitor) {
+ this.singleRoot.accept(visitor);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Utility Classes
+ // --------------------------------------------------------------------------------------------
+
+ private static final class WorksetOpDescriptor extends OperatorDescriptorDual {
+
+ private WorksetOpDescriptor(FieldList solutionSetKeys) {
+ super(solutionSetKeys, null);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.NONE;
+ }
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+ RequestedGlobalProperties partitionedGp = new RequestedGlobalProperties();
+ partitionedGp.setHashPartitioned(this.keys1);
+ return Collections.singletonList(new GlobalPropertiesPair(partitionedGp, new RequestedGlobalProperties()));
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new LocalPropertiesPair(
+ new RequestedLocalProperties(), new RequestedLocalProperties()));
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static class SingleRootJoiner extends TwoInputNode {
+
+ SingleRootJoiner() {
+ super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));
+
+ setDegreeOfParallelism(1);
+ }
+
+ public void setInputs(DagConnection input1, DagConnection input2) {
+ this.input1 = input1;
+ this.input2 = input2;
+ }
+
+ @Override
+ public String getName() {
+ return "Internal Utility Node";
+ }
+
+ @Override
+ protected List<OperatorDescriptorDual> getPossibleProperties() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
+ // no estimates are needed here
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
new file mode 100644
index 0000000..3b05aba
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/WorksetNode.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dag;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
+import org.apache.flink.optimizer.DataStatistics;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.PlanNode;
+import org.apache.flink.optimizer.plan.WorksetPlanNode;
+
+/**
+ * The optimizer's internal representation of the partial solution that is input to a bulk iteration.
+ */
+public class WorksetNode extends AbstractPartialSolutionNode {
+
+ private final WorksetIterationNode iterationNode;
+
+
+ public WorksetNode(WorksetPlaceHolder<?> psph, WorksetIterationNode iterationNode) {
+ super(psph);
+ this.iterationNode = iterationNode;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
+ if (this.cachedPlans != null) {
+ throw new IllegalStateException();
+ } else {
+ WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
+ this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
+ }
+ }
+
+ public WorksetPlanNode getCurrentWorksetPlanNode() {
+ if (this.cachedPlans != null) {
+ return (WorksetPlanNode) this.cachedPlans.get(0);
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ public WorksetIterationNode getIterationNode() {
+ return this.iterationNode;
+ }
+
+ @Override
+ public void computeOutputEstimates(DataStatistics statistics) {
+ copyEstimates(this.iterationNode.getInitialWorksetPredecessorNode());
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the contract object for this data source node.
+ *
+ * @return The contract.
+ */
+ @Override
+ public WorksetPlaceHolder<?> getOperator() {
+ return (WorksetPlaceHolder<?>) super.getOperator();
+ }
+
+ @Override
+ public String getName() {
+ return "Workset";
+ }
+
+ @Override
+ public void computeUnclosedBranchStack() {
+ if (this.openBranches != null) {
+ return;
+ }
+
+ DagConnection worksetInput = this.iterationNode.getSecondIncomingConnection();
+ OptimizerNode worksetSource = worksetInput.getSource();
+
+ addClosedBranches(worksetSource.closedBranchingNodes);
+ List<UnclosedBranchDescriptor> fromInput = worksetSource.getBranchesForParent(worksetInput);
+ this.openBranches = (fromInput == null || fromInput.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : fromInput;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
new file mode 100644
index 0000000..57ba29d
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
@@ -0,0 +1,500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents global properties of the data at a certain point in the plan.
+ * Global properties are properties that describe data across different partitions, such as
+ * whether the data is hash partitioned, range partitioned, replicated, etc.
+ */
+public class GlobalProperties implements Cloneable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(GlobalProperties.class);
+
+ private PartitioningProperty partitioning; // the type partitioning
+
+ private FieldList partitioningFields; // the fields which are partitioned
+
+ private Ordering ordering; // order of the partitioned fields, if it is an ordered (range) range partitioning
+
+ private Set<FieldSet> uniqueFieldCombinations;
+
+ private Partitioner<?> customPartitioner;
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Initializes the global properties with no partitioning.
+ */
+ public GlobalProperties() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Sets this global properties to represent a hash partitioning.
+ *
+ * @param partitionedFields The key fields on which the data is hash partitioned.
+ */
+ public void setHashPartitioned(FieldList partitionedFields) {
+ if (partitionedFields == null) {
+ throw new NullPointerException();
+ }
+
+ this.partitioning = PartitioningProperty.HASH_PARTITIONED;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ }
+
+
+ public void setRangePartitioned(Ordering ordering) {
+ if (ordering == null) {
+ throw new NullPointerException();
+ }
+
+ this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
+ this.ordering = ordering;
+ this.partitioningFields = ordering.getInvolvedIndexes();
+ }
+
+ public void setAnyPartitioning(FieldList partitionedFields) {
+ if (partitionedFields == null) {
+ throw new NullPointerException();
+ }
+
+ this.partitioning = PartitioningProperty.ANY_PARTITIONING;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ }
+
+ public void setRandomPartitioned() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setFullyReplicated() {
+ this.partitioning = PartitioningProperty.FULL_REPLICATION;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setForcedRebalanced() {
+ this.partitioning = PartitioningProperty.FORCED_REBALANCED;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setCustomPartitioned(FieldList partitionedFields, Partitioner<?> partitioner) {
+ if (partitionedFields == null || partitioner == null) {
+ throw new NullPointerException();
+ }
+
+ this.partitioning = PartitioningProperty.CUSTOM_PARTITIONING;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ this.customPartitioner = partitioner;
+ }
+
+ public void addUniqueFieldCombination(FieldSet fields) {
+ if (fields == null) {
+ return;
+ }
+ if (this.uniqueFieldCombinations == null) {
+ this.uniqueFieldCombinations = new HashSet<FieldSet>();
+ }
+ this.uniqueFieldCombinations.add(fields);
+ }
+
+ public void clearUniqueFieldCombinations() {
+ if (this.uniqueFieldCombinations != null) {
+ this.uniqueFieldCombinations = null;
+ }
+ }
+
+ public Set<FieldSet> getUniqueFieldCombination() {
+ return this.uniqueFieldCombinations;
+ }
+
+ public FieldList getPartitioningFields() {
+ return this.partitioningFields;
+ }
+
+ public Ordering getPartitioningOrdering() {
+ return this.ordering;
+ }
+
+ public PartitioningProperty getPartitioning() {
+ return this.partitioning;
+ }
+
+ public Partitioner<?> getCustomPartitioner() {
+ return this.customPartitioner;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public boolean isPartitionedOnFields(FieldSet fields) {
+ if (this.partitioning.isPartitionedOnKey() && fields.isValidSubset(this.partitioningFields)) {
+ return true;
+ } else if (this.uniqueFieldCombinations != null) {
+ for (FieldSet set : this.uniqueFieldCombinations) {
+ if (fields.isValidSubset(set)) {
+ return true;
+ }
+ }
+ return false;
+ } else {
+ return false;
+ }
+ }
+
+ public boolean isExactlyPartitionedOnFields(FieldList fields) {
+ return this.partitioning.isPartitionedOnKey() && fields.isExactMatch(this.partitioningFields);
+ }
+
+ public boolean matchesOrderedPartitioning(Ordering o) {
+ if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
+ if (this.ordering.getNumberOfFields() > o.getNumberOfFields()) {
+ return false;
+ }
+
+ for (int i = 0; i < this.ordering.getNumberOfFields(); i++) {
+ if (this.ordering.getFieldNumber(i) != o.getFieldNumber(i)) {
+ return false;
+ }
+
+ // if this one request no order, everything is good
+ final Order oo = o.getOrder(i);
+ final Order to = this.ordering.getOrder(i);
+ if (oo != Order.NONE) {
+ if (oo == Order.ANY) {
+ // if any order is requested, any not NONE order is good
+ if (to == Order.NONE) {
+ return false;
+ }
+ } else if (oo != to) {
+ // the orders must be equal
+ return false;
+ }
+ }
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public boolean isFullyReplicated() {
+ return this.partitioning == PartitioningProperty.FULL_REPLICATION;
+ }
+
+ /**
+ * Checks, if the properties in this object are trivial, i.e. only standard values.
+ */
+ public boolean isTrivial() {
+ return partitioning == PartitioningProperty.RANDOM_PARTITIONED;
+ }
+
+ /**
+ * This method resets the properties to a state where no properties are given.
+ */
+ public void reset() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ this.ordering = null;
+ this.partitioningFields = null;
+ }
+
+ /**
+ * Filters these GlobalProperties by the fields that are forwarded to the output
+ * as described by the SemanticProperties.
+ *
+ * @param props The semantic properties holding information about forwarded fields.
+ * @param input The index of the input.
+ * @return The filtered GlobalProperties
+ */
+ public GlobalProperties filterBySemanticProperties(SemanticProperties props, int input) {
+
+ if (props == null) {
+ throw new NullPointerException("SemanticProperties may not be null.");
+ }
+
+ GlobalProperties gp = new GlobalProperties();
+
+ // filter partitioning
+ switch(this.partitioning) {
+ case RANGE_PARTITIONED:
+ // check if ordering is preserved
+ Ordering newOrdering = new Ordering();
+ for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
+ int sourceField = this.ordering.getInvolvedIndexes().get(i);
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+
+ if (targetField == null || targetField.size() == 0) {
+ // partitioning is destroyed
+ newOrdering = null;
+ break;
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
+ }
+ }
+ if(newOrdering != null) {
+ gp.partitioning = PartitioningProperty.RANGE_PARTITIONED;
+ gp.ordering = newOrdering;
+ gp.partitioningFields = newOrdering.getInvolvedIndexes();
+ }
+ break;
+ case HASH_PARTITIONED:
+ case ANY_PARTITIONING:
+ case CUSTOM_PARTITIONING:
+ FieldList newPartitioningFields = new FieldList();
+ for (int sourceField : this.partitioningFields) {
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+
+ if (targetField == null || targetField.size() == 0) {
+ newPartitioningFields = null;
+ break;
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newPartitioningFields = newPartitioningFields.addField(targetField.toArray()[0]);
+ }
+ }
+ if(newPartitioningFields != null) {
+ gp.partitioning = this.partitioning;
+ gp.partitioningFields = newPartitioningFields;
+ gp.customPartitioner = this.customPartitioner;
+ }
+ break;
+ case FORCED_REBALANCED:
+ case FULL_REPLICATION:
+ case RANDOM_PARTITIONED:
+ gp.partitioning = this.partitioning;
+ break;
+ default:
+ throw new RuntimeException("Unknown partitioning type.");
+ }
+
+ // filter unique field combinations
+ if (this.uniqueFieldCombinations != null) {
+ Set<FieldSet> newUniqueFieldCombinations = new HashSet<FieldSet>();
+ for (FieldSet fieldCombo : this.uniqueFieldCombinations) {
+ FieldSet newFieldCombo = new FieldSet();
+ for (Integer sourceField : fieldCombo) {
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+
+ if (targetField == null || targetField.size() == 0) {
+ newFieldCombo = null;
+ break;
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newFieldCombo = newFieldCombo.addField(targetField.toArray()[0]);
+ }
+ }
+ if (newFieldCombo != null) {
+ newUniqueFieldCombinations.add(newFieldCombo);
+ }
+ }
+ if(!newUniqueFieldCombinations.isEmpty()) {
+ gp.uniqueFieldCombinations = newUniqueFieldCombinations;
+ }
+ }
+
+ return gp;
+ }
+
+
+ public void parameterizeChannel(Channel channel, boolean globalDopChange,
+ ExecutionMode exchangeMode, boolean breakPipeline) {
+
+ ShipStrategyType shipType;
+ FieldList partitionKeys;
+ boolean[] sortDirection;
+ Partitioner<?> partitioner;
+
+ switch (this.partitioning) {
+ case RANDOM_PARTITIONED:
+ shipType = globalDopChange ? ShipStrategyType.PARTITION_RANDOM : ShipStrategyType.FORWARD;
+ partitionKeys = null;
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case FULL_REPLICATION:
+ shipType = ShipStrategyType.BROADCAST;
+ partitionKeys = null;
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case ANY_PARTITIONING:
+ case HASH_PARTITIONED:
+ shipType = ShipStrategyType.PARTITION_HASH;
+ partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case RANGE_PARTITIONED:
+ shipType = ShipStrategyType.PARTITION_RANGE;
+ partitionKeys = this.ordering.getInvolvedIndexes();
+ sortDirection = this.ordering.getFieldSortDirections();
+ partitioner = null;
+ break;
+
+ case FORCED_REBALANCED:
+ shipType = ShipStrategyType.PARTITION_RANDOM;
+ partitionKeys = null;
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case CUSTOM_PARTITIONING:
+ shipType = ShipStrategyType.PARTITION_CUSTOM;
+ partitionKeys = this.partitioningFields;
+ sortDirection = null;
+ partitioner = this.customPartitioner;
+ break;
+
+ default:
+ throw new CompilerException("Unsupported partitioning strategy");
+ }
+
+ DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
+ channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((partitioning == null) ? 0 : partitioning.ordinal());
+ result = prime * result + ((partitioningFields == null) ? 0 : partitioningFields.hashCode());
+ result = prime * result + ((ordering == null) ? 0 : ordering.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof GlobalProperties) {
+ final GlobalProperties other = (GlobalProperties) obj;
+ return (this.partitioning == other.partitioning)
+ && (this.ordering == other.ordering || (this.ordering != null && this.ordering.equals(other.ordering)))
+ && (this.partitioningFields == other.partitioningFields ||
+ (this.partitioningFields != null && this.partitioningFields.equals(other.partitioningFields)))
+ && (this.uniqueFieldCombinations == other.uniqueFieldCombinations ||
+ (this.uniqueFieldCombinations != null && this.uniqueFieldCombinations.equals(other.uniqueFieldCombinations)));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder bld = new StringBuilder(
+ "GlobalProperties [partitioning=" + partitioning +
+ (this.partitioningFields == null ? "" : ", on fields " + this.partitioningFields) +
+ (this.ordering == null ? "" : ", with ordering " + this.ordering));
+
+ if (this.uniqueFieldCombinations == null) {
+ bld.append(']');
+ } else {
+ bld.append(" - Unique field groups: ");
+ bld.append(this.uniqueFieldCombinations);
+ bld.append(']');
+ }
+ return bld.toString();
+ }
+
+ @Override
+ public GlobalProperties clone() {
+ final GlobalProperties newProps = new GlobalProperties();
+ newProps.partitioning = this.partitioning;
+ newProps.partitioningFields = this.partitioningFields;
+ newProps.ordering = this.ordering;
+ newProps.customPartitioner = this.customPartitioner;
+ newProps.uniqueFieldCombinations = this.uniqueFieldCombinations == null ? null : new HashSet<FieldSet>(this.uniqueFieldCombinations);
+ return newProps;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static GlobalProperties combine(GlobalProperties gp1, GlobalProperties gp2) {
+ if (gp1.isFullyReplicated()) {
+ if (gp2.isFullyReplicated()) {
+ return new GlobalProperties();
+ } else {
+ return gp2;
+ }
+ } else if (gp2.isFullyReplicated()) {
+ return gp1;
+ } else if (gp1.ordering != null) {
+ return gp1;
+ } else if (gp2.ordering != null) {
+ return gp2;
+ } else if (gp1.partitioningFields != null) {
+ return gp1;
+ } else if (gp2.partitioningFields != null) {
+ return gp2;
+ } else if (gp1.uniqueFieldCombinations != null) {
+ return gp1;
+ } else if (gp2.uniqueFieldCombinations != null) {
+ return gp2;
+ } else if (gp1.getPartitioning().isPartitioned()) {
+ return gp1;
+ } else if (gp2.getPartitioning().isPartitioned()) {
+ return gp2;
+ } else {
+ return gp1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
new file mode 100644
index 0000000..6946641
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/InterestingProperties.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+
+/**
+ * Interesting properties are propagated from parent operators to child operators. They tell the child
+ * what data properties would help the parent in operating in a cheaper fashion. A reduce operator, for
+ * example, tells its child that partitioned data would help. If the child is a join operator, it can use
+ * that knowledge to favor strategies that leave the data in a partitioned form.
+ *
+ * More on optimization with interesting properties can be found in the works on
+ * the volcano- and cascades optimizer framework.
+ */
+public class InterestingProperties implements Cloneable {
+
+ private Set<RequestedGlobalProperties> globalProps; // the global properties, i.e. properties across partitions
+
+ private Set<RequestedLocalProperties> localProps; // the local properties, i.e. properties within partitions
+
+ // ------------------------------------------------------------------------
+
+ public InterestingProperties() {
+ this.globalProps = new HashSet<RequestedGlobalProperties>();
+ this.localProps = new HashSet<RequestedLocalProperties>();
+ }
+
+ /**
+ * Private constructor for cloning purposes.
+ *
+ * @param globalProps The global properties for this new object.
+ * @param localProps The local properties for this new object.
+ */
+ private InterestingProperties(Set<RequestedGlobalProperties> globalProps, Set<RequestedLocalProperties> localProps) {
+ this.globalProps = globalProps;
+ this.localProps = localProps;
+ }
+
+ // ------------------------------------------------------------------------
+
+ public void addGlobalProperties(RequestedGlobalProperties props) {
+ this.globalProps.add(props);
+ }
+
+ public void addLocalProperties(RequestedLocalProperties props) {
+ this.localProps.add(props);
+ }
+
+ public void addInterestingProperties(InterestingProperties other) {
+ this.globalProps.addAll(other.globalProps);
+ this.localProps.addAll(other.localProps);
+ }
+
+ /**
+ * Gets the interesting local properties.
+ *
+ * @return The interesting local properties.
+ */
+ public Set<RequestedLocalProperties> getLocalProperties() {
+ return this.localProps;
+ }
+
+ /**
+ * Gets the interesting global properties.
+ *
+ * @return The interesting global properties.
+ */
+ public Set<RequestedGlobalProperties> getGlobalProperties() {
+ return this.globalProps;
+ }
+
+ public InterestingProperties filterByCodeAnnotations(OptimizerNode node, int input) {
+ InterestingProperties iProps = new InterestingProperties();
+ SemanticProperties props;
+ if (node instanceof SingleInputNode || node instanceof TwoInputNode) {
+ props = node.getSemanticProperties();
+ } else {
+ props = new SemanticProperties.EmptySemanticProperties();
+ }
+
+ for (RequestedGlobalProperties rgp : this.globalProps) {
+ RequestedGlobalProperties filtered = rgp.filterBySemanticProperties(props, input);
+ if (filtered != null && !filtered.isTrivial()) {
+ iProps.addGlobalProperties(filtered);
+ }
+ }
+ for (RequestedLocalProperties rlp : this.localProps) {
+ RequestedLocalProperties filtered = rlp.filterBySemanticProperties(props, input);
+ if (filtered != null && !filtered.isTrivial()) {
+ iProps.addLocalProperties(filtered);
+ }
+ }
+ return iProps;
+ }
+
+ public void dropTrivials() {
+ for (Iterator<RequestedGlobalProperties> iter = this.globalProps.iterator(); iter.hasNext();) {
+ RequestedGlobalProperties gp = iter.next();
+ if (gp.isTrivial()) {
+ iter.remove();
+ break;
+ }
+ }
+
+ for (Iterator<RequestedLocalProperties> iter = this.localProps.iterator(); iter.hasNext();) {
+ RequestedLocalProperties lp = iter.next();
+ if (lp.isTrivial()) {
+ iter.remove();
+ break;
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((globalProps == null) ? 0 : globalProps.hashCode());
+ result = prime * result + ((localProps == null) ? 0 : localProps.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof InterestingProperties) {
+ InterestingProperties other = (InterestingProperties) obj;
+ return this.globalProps.equals(other.globalProps) &&
+ this.localProps.equals(other.localProps);
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "InterestingProperties [globalProps=" + this.globalProps +
+ ", localProps=" + this.localProps + " ]";
+ }
+
+ @Override
+ public InterestingProperties clone() {
+ HashSet<RequestedGlobalProperties> globalProps = new HashSet<RequestedGlobalProperties>();
+ for (RequestedGlobalProperties p : this.globalProps) {
+ globalProps.add(p.clone());
+ }
+ HashSet<RequestedLocalProperties> localProps = new HashSet<RequestedLocalProperties>();
+ for (RequestedLocalProperties p : this.localProps) {
+ localProps.add(p.clone());
+ }
+
+ return new InterestingProperties(globalProps, localProps);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
new file mode 100644
index 0000000..e0231aa
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/LocalProperties.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.dataproperties;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents local properties of the data. A local property is a property that exists
+ * within the data of a single partition, such as sort order, or data grouping.
+ */
+public class LocalProperties implements Cloneable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(GlobalProperties.class);
+
+ public static final LocalProperties EMPTY = new LocalProperties();
+
+ // --------------------------------------------------------------------------------------------
+
+ private Ordering ordering; // order inside a partition, null if not ordered
+
+ private FieldList groupedFields; // fields by which the stream is grouped. null if not grouped.
+
+ private Set<FieldSet> uniqueFields; // fields whose value combination is unique in the stream
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Default constructor for trivial local properties. No order, no grouping, no uniqueness.
+ */
+ public LocalProperties() {}
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Gets the key order.
+ *
+ * @return The key order, or <code>null</code> if nothing is ordered.
+ */
+ public Ordering getOrdering() {
+ return ordering;
+ }
+
+ /**
+ * Gets the grouped fields.
+ *
+ * @return The grouped fields, or <code>null</code> if nothing is grouped.
+ */
+ public FieldList getGroupedFields() {
+ return this.groupedFields;
+ }
+
+ /**
+ * Gets the fields whose combination is unique within the data set.
+ *
+ * @return The unique field combination, or <code>null</code> if nothing is unique.
+ */
+ public Set<FieldSet> getUniqueFields() {
+ return this.uniqueFields;
+ }
+
+ /**
+ * Checks whether the given set of fields is unique, as specified in these local properties.
+ *
+ * @param set The set to check.
+ * @return True, if the given column combination is unique, false if not.
+ */
+ public boolean areFieldsUnique(FieldSet set) {
+ return this.uniqueFields != null && this.uniqueFields.contains(set);
+ }
+
+ /**
+ * Adds a combination of fields that are unique in these data properties.
+ *
+ * @param uniqueFields The fields that are unique in these data properties.
+ */
+ public LocalProperties addUniqueFields(FieldSet uniqueFields) {
+ LocalProperties copy = clone();
+
+ if (copy.uniqueFields == null) {
+ copy.uniqueFields = new HashSet<FieldSet>();
+ }
+ copy.uniqueFields.add(uniqueFields);
+ return copy;
+ }
+
+ public LocalProperties clearUniqueFieldSets() {
+ if (this.uniqueFields == null || this.uniqueFields.isEmpty()) {
+ return this;
+ } else {
+ LocalProperties copy = new LocalProperties();
+ copy.ordering = this.ordering;
+ copy.groupedFields = this.groupedFields;
+ return copy;
+ }
+ }
+
+ /**
+ * Checks, if the properties in this object are trivial, i.e. only standard values.
+ */
+ public boolean isTrivial() {
+ return ordering == null && this.groupedFields == null && this.uniqueFields == null;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Filters these LocalProperties by the fields that are forwarded to the output
+ * as described by the SemanticProperties.
+ *
+ * @param props The semantic properties holding information about forwarded fields.
+ * @param input The index of the input.
+ * @return The filtered LocalProperties
+ */
+ public LocalProperties filterBySemanticProperties(SemanticProperties props, int input) {
+
+ if (props == null) {
+ throw new NullPointerException("SemanticProperties may not be null.");
+ }
+
+ LocalProperties returnProps = new LocalProperties();
+
+ // check if sorting is preserved
+ if (this.ordering != null) {
+ Ordering newOrdering = new Ordering();
+
+ for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
+ int sourceField = this.ordering.getInvolvedIndexes().get(i);
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+ if (targetField == null || targetField.size() == 0) {
+ if (i == 0) {
+ // order fully destroyed
+ newOrdering = null;
+ break;
+ } else {
+ // order partially preserved
+ break;
+ }
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newOrdering.appendOrdering(targetField.toArray()[0], this.ordering.getType(i), this.ordering.getOrder(i));
+ }
+ }
+
+ returnProps.ordering = newOrdering;
+ if (newOrdering != null) {
+ returnProps.groupedFields = newOrdering.getInvolvedIndexes();
+ } else {
+ returnProps.groupedFields = null;
+ }
+ }
+ // check if grouping is preserved
+ else if (this.groupedFields != null) {
+ FieldList newGroupedFields = new FieldList();
+
+ for (Integer sourceField : this.groupedFields) {
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+ if (targetField == null || targetField.size() == 0) {
+ newGroupedFields = null;
+ break;
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newGroupedFields = newGroupedFields.addField(targetField.toArray()[0]);
+ }
+ }
+ returnProps.groupedFields = newGroupedFields;
+ }
+
+ if (this.uniqueFields != null) {
+ Set<FieldSet> newUniqueFields = new HashSet<FieldSet>();
+ for (FieldSet fields : this.uniqueFields) {
+ FieldSet newFields = new FieldSet();
+ for (Integer sourceField : fields) {
+ FieldSet targetField = props.getForwardingTargetFields(input, sourceField);
+
+ if (targetField == null || targetField.size() == 0) {
+ newFields = null;
+ break;
+ } else {
+ // use any field of target fields for now. We should use something like field equivalence sets in the future.
+ if(targetField.size() > 1) {
+ LOG.warn("Found that a field is forwarded to more than one target field in " +
+ "semantic forwarded field information. Will only use the field with the lowest index.");
+ }
+ newFields = newFields.addField(targetField.toArray()[0]);
+ }
+ }
+ if (newFields != null) {
+ newUniqueFields.add(newFields);
+ }
+ }
+
+ if (!newUniqueFields.isEmpty()) {
+ returnProps.uniqueFields = newUniqueFields;
+ } else {
+ returnProps.uniqueFields = null;
+ }
+ }
+
+ return returnProps;
+ }
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (this.ordering == null ? 0 : this.ordering.hashCode());
+ result = prime * result + (this.groupedFields == null ? 0 : this.groupedFields.hashCode());
+ result = prime * result + (this.uniqueFields == null ? 0 : this.uniqueFields.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof LocalProperties) {
+ final LocalProperties other = (LocalProperties) obj;
+ return (ordering == other.getOrdering() || (ordering != null && ordering.equals(other.getOrdering()))) &&
+ (groupedFields == other.getGroupedFields() || (groupedFields != null && groupedFields.equals(other.getGroupedFields()))) &&
+ (uniqueFields == other.getUniqueFields() || (uniqueFields != null && uniqueFields.equals(other.getUniqueFields())));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LocalProperties [ordering=" + this.ordering + ", grouped=" + this.groupedFields
+ + ", unique=" + this.uniqueFields + "]";
+ }
+
+ @Override
+ public LocalProperties clone() {
+ LocalProperties copy = new LocalProperties();
+ copy.ordering = this.ordering;
+ copy.groupedFields = this.groupedFields;
+ copy.uniqueFields = (this.uniqueFields == null ? null : new HashSet<FieldSet>(this.uniqueFields));
+ return copy;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static LocalProperties combine(LocalProperties lp1, LocalProperties lp2) {
+ if (lp1.ordering != null) {
+ return lp1;
+ } else if (lp2.ordering != null) {
+ return lp2;
+ } else if (lp1.groupedFields != null) {
+ return lp1;
+ } else if (lp2.groupedFields != null) {
+ return lp2;
+ } else if (lp1.uniqueFields != null && !lp1.uniqueFields.isEmpty()) {
+ return lp1;
+ } else if (lp2.uniqueFields != null && !lp2.uniqueFields.isEmpty()) {
+ return lp2;
+ } else {
+ return lp1;
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static LocalProperties forOrdering(Ordering o) {
+ LocalProperties props = new LocalProperties();
+ props.ordering = o;
+ props.groupedFields = o.getInvolvedIndexes();
+ return props;
+ }
+
+ public static LocalProperties forGrouping(FieldList groupedFields) {
+ LocalProperties props = new LocalProperties();
+ props.groupedFields = groupedFields;
+ return props;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
new file mode 100644
index 0000000..5e06dd3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/PartitioningProperty.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+/**
+ * An enumeration of the the different types of distributing data across partitions or
+ * parallel workers.
+ */
+public enum PartitioningProperty {
+
+ /**
+ * Any possible way of data distribution, including random partitioning and full replication.
+ */
+ ANY_DISTRIBUTION,
+
+ /**
+ * A random disjunct (non-replicated) data distribution, where each datum is contained in one partition only.
+ * This is for example the result of parallel scans of data in a file system like HDFS,
+ * or the result of a round-robin data distribution.
+ */
+ RANDOM_PARTITIONED,
+
+ /**
+ * A hash partitioning on a certain key.
+ */
+ HASH_PARTITIONED,
+
+ /**
+ * A range partitioning on a certain key.
+ */
+ RANGE_PARTITIONED,
+
+ /**
+ * A not further specified partitioning on a key (hash-, or range partitioning, or some other scheme even).
+ */
+ ANY_PARTITIONING,
+
+ /**
+ *Full replication of the data to each parallel instance.
+ */
+ FULL_REPLICATION,
+
+ /**
+ * A forced even re-balancing. All partitions are guaranteed to have almost the same number of records.
+ */
+ FORCED_REBALANCED,
+
+ /**
+ * A custom partitioning, accompanied by a {@link org.apache.flink.api.common.functions.Partitioner}.
+ */
+ CUSTOM_PARTITIONING;
+
+ /**
+ * Checks, if this property represents in fact a partitioning. That is,
+ * whether this property is not equal to <tt>PartitionProperty.FULL_REPLICATION</tt>.
+ *
+ * @return True, if this enum constant is unequal to <tt>PartitionProperty.FULL_REPLICATION</tt>,
+ * false otherwise.
+ */
+ public boolean isPartitioned() {
+ return this != FULL_REPLICATION && this != FORCED_REBALANCED && this != ANY_DISTRIBUTION;
+ }
+
+ /**
+ * Checks, if this property represents a full replication.
+ *
+ * @return True, if this enum constant is equal to <tt>PartitionProperty.FULL_REPLICATION</tt>,
+ * false otherwise.
+ */
+ public boolean isReplication() {
+ return this == FULL_REPLICATION;
+ }
+
+ /**
+ * Checks if this property presents a partitioning that is not random, but on a partitioning key.
+ *
+ * @return True, if the data is partitioned on a key.
+ */
+ public boolean isPartitionedOnKey() {
+ return isPartitioned() && this != RANDOM_PARTITIONED;
+ }
+
+ /**
+ * Checks, if this property represents a partitioning that is computable.
+ * A computable partitioning can be recreated through an algorithm. If two sets of data are to
+ * be co-partitioned, it is crucial, that the partitioning schemes are computable.
+ * <p>
+ * Examples for computable partitioning schemes are hash- or range-partitioning. An example for a non-computable
+ * partitioning is the implicit partitioning that exists though a globally unique key.
+ *
+ * @return True, if this enum constant is a re-computable partitioning.
+ */
+ public boolean isComputablyPartitioned() {
+ return this == HASH_PARTITIONED || this == RANGE_PARTITIONED || this == CUSTOM_PARTITIONING;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
new file mode 100644
index 0000000..8c3f6bd
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/RequestedGlobalProperties.java
@@ -0,0 +1,486 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.dataproperties;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.SemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+
+/**
+ * This class represents the global properties of the data that are requested by an operator.
+ * Operators request the global properties they need for correct execution. This list is an example of global
+ * properties requested by certain operators:
+ * <ul>
+ * <li>"groupBy/reduce" will request the data to be partitioned in some way after the key fields.</li>
+ * <li>"map" will request the data to be in an arbitrary distribution - it has no prerequisites</li>
+ * <li>"join" will request certain properties for each input. This class represents the properties
+ * on an input alone. The properties may be partitioning on the key fields, or a combination of
+ * replication on one input and anything-but-replication on the other input.</li>
+ * </ul>
+ */
+public final class RequestedGlobalProperties implements Cloneable {
+
+ private PartitioningProperty partitioning; // the type partitioning
+
+ private FieldSet partitioningFields; // the fields which are partitioned
+
+ private Ordering ordering; // order of the partitioned fields, if it is an ordered (range) range partitioning
+
+ private DataDistribution dataDistribution; // optional data distribution, for a range partitioning
+
+ private Partitioner<?> customPartitioner; // optional, partitioner for custom partitioning
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Initializes the global properties with no partitioning.
+ */
+ public RequestedGlobalProperties() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Sets these properties to request a hash partitioning on the given fields.
+ *
+ * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
+ * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
+ * then only an exact partitioning on the fields matches this requested partitioning.
+ *
+ * @param partitionedFields The key fields for the partitioning.
+ */
+ public void setHashPartitioned(FieldSet partitionedFields) {
+ if (partitionedFields == null) {
+ throw new NullPointerException();
+ }
+ this.partitioning = PartitioningProperty.HASH_PARTITIONED;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ }
+
+
+ public void setRangePartitioned(Ordering ordering) {
+ this.setRangePartitioned(ordering, null);
+ }
+
+ public void setRangePartitioned(Ordering ordering, DataDistribution dataDistribution) {
+ if (ordering == null) {
+ throw new NullPointerException();
+ }
+ this.partitioning = PartitioningProperty.RANGE_PARTITIONED;
+ this.ordering = ordering;
+ this.partitioningFields = null;
+ this.dataDistribution = dataDistribution;
+ }
+
+ /**
+ * Sets these properties to request some partitioning on the given fields. This will allow
+ * both hash partitioning and range partitioning to match.
+ *
+ * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
+ * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
+ * then only an exact partitioning on the fields matches this requested partitioning.
+ *
+ * @param partitionedFields The key fields for the partitioning.
+ */
+ public void setAnyPartitioning(FieldSet partitionedFields) {
+ if (partitionedFields == null) {
+ throw new NullPointerException();
+ }
+ this.partitioning = PartitioningProperty.ANY_PARTITIONING;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ }
+
+ public void setRandomPartitioning() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setAnyDistribution() {
+ this.partitioning = PartitioningProperty.ANY_DISTRIBUTION;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setFullyReplicated() {
+ this.partitioning = PartitioningProperty.FULL_REPLICATION;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ public void setForceRebalancing() {
+ this.partitioning = PartitioningProperty.FORCED_REBALANCED;
+ this.partitioningFields = null;
+ this.ordering = null;
+ }
+
+ /**
+ * Sets these properties to request a custom partitioning with the given {@link Partitioner} instance.
+ *
+ * If the fields are provided as {@link FieldSet}, then any permutation of the fields is a
+ * valid partitioning, including subsets. If the fields are given as a {@link FieldList},
+ * then only an exact partitioning on the fields matches this requested partitioning.
+ *
+ * @param partitionedFields The key fields for the partitioning.
+ */
+ public void setCustomPartitioned(FieldSet partitionedFields, Partitioner<?> partitioner) {
+ if (partitionedFields == null || partitioner == null) {
+ throw new NullPointerException();
+ }
+
+ this.partitioning = PartitioningProperty.CUSTOM_PARTITIONING;
+ this.partitioningFields = partitionedFields;
+ this.ordering = null;
+ this.customPartitioner = partitioner;
+ }
+
+ /**
+ * Gets the partitioning property.
+ *
+ * @return The partitioning property.
+ */
+ public PartitioningProperty getPartitioning() {
+ return partitioning;
+ }
+
+ /**
+ * Gets the fields on which the data is partitioned.
+ *
+ * @return The partitioning fields.
+ */
+ public FieldSet getPartitionedFields() {
+ return this.partitioningFields;
+ }
+
+ /**
+ * Gets the key order.
+ *
+ * @return The key order.
+ */
+ public Ordering getOrdering() {
+ return this.ordering;
+ }
+
+ /**
+ * Gets the data distribution.
+ *
+ * @return The data distribution.
+ */
+ public DataDistribution getDataDistribution() {
+ return this.dataDistribution;
+ }
+
+ /**
+ * Gets the custom partitioner associated with these properties.
+ *
+ * @return The custom partitioner associated with these properties.
+ */
+ public Partitioner<?> getCustomPartitioner() {
+ return customPartitioner;
+ }
+
+ /**
+ * Checks, if the properties in this object are trivial, i.e. only standard values.
+ */
+ public boolean isTrivial() {
+ return this.partitioning == null || this.partitioning == PartitioningProperty.RANDOM_PARTITIONED;
+ }
+
+ /**
+ * This method resets the properties to a state where no properties are given.
+ */
+ public void reset() {
+ this.partitioning = PartitioningProperty.RANDOM_PARTITIONED;
+ this.ordering = null;
+ this.partitioningFields = null;
+ this.dataDistribution = null;
+ this.customPartitioner = null;
+ }
+
+ /**
+ * Filters these properties by what can be preserved by the given SemanticProperties when propagated down
+ * to the given input.
+ *
+ * @param props The SemanticProperties which define which fields are preserved.
+ * @param input The index of the operator's input.
+ * @return The filtered RequestedGlobalProperties
+ */
+ public RequestedGlobalProperties filterBySemanticProperties(SemanticProperties props, int input) {
+ // no semantic properties available. All global properties are filtered.
+ if (props == null) {
+ throw new NullPointerException("SemanticProperties may not be null.");
+ }
+
+ RequestedGlobalProperties rgProp = new RequestedGlobalProperties();
+
+ switch(this.partitioning) {
+ case FULL_REPLICATION:
+ case FORCED_REBALANCED:
+ case CUSTOM_PARTITIONING:
+ case RANDOM_PARTITIONED:
+ case ANY_DISTRIBUTION:
+ // make sure that certain properties are not pushed down
+ return null;
+ case HASH_PARTITIONED:
+ case ANY_PARTITIONING:
+ FieldSet newFields;
+ if(this.partitioningFields instanceof FieldList) {
+ newFields = new FieldList();
+ } else {
+ newFields = new FieldSet();
+ }
+
+ for (Integer targetField : this.partitioningFields) {
+ int sourceField = props.getForwardingSourceField(input, targetField);
+ if (sourceField >= 0) {
+ newFields = newFields.addField(sourceField);
+ } else {
+ // partial partitionings are not preserved to avoid skewed partitioning
+ return null;
+ }
+ }
+ rgProp.partitioning = this.partitioning;
+ rgProp.partitioningFields = newFields;
+ return rgProp;
+ case RANGE_PARTITIONED:
+ // range partitioning
+ Ordering newOrdering = new Ordering();
+ for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) {
+ int value = this.ordering.getInvolvedIndexes().get(i);
+ int sourceField = props.getForwardingSourceField(input, value);
+ if (sourceField >= 0) {
+ newOrdering.appendOrdering(sourceField, this.ordering.getType(i), this.ordering.getOrder(i));
+ } else {
+ return null;
+ }
+ }
+ rgProp.partitioning = this.partitioning;
+ rgProp.ordering = newOrdering;
+ rgProp.dataDistribution = this.dataDistribution;
+ return rgProp;
+ default:
+ throw new RuntimeException("Unknown partitioning type encountered.");
+ }
+ }
+
+ /**
+ * Checks, if this set of interesting properties, is met by the given
+ * produced properties.
+ *
+ * @param props The properties for which to check whether they meet these properties.
+ * @return True, if the properties are met, false otherwise.
+ */
+ public boolean isMetBy(GlobalProperties props) {
+ if (this.partitioning == PartitioningProperty.ANY_DISTRIBUTION) {
+ return true;
+ } else if (this.partitioning == PartitioningProperty.FULL_REPLICATION) {
+ return props.isFullyReplicated();
+ }
+ else if (props.isFullyReplicated()) {
+ return false;
+ }
+ else if (this.partitioning == PartitioningProperty.RANDOM_PARTITIONED) {
+ return true;
+ }
+ else if (this.partitioning == PartitioningProperty.ANY_PARTITIONING) {
+ return checkCompatiblePartitioningFields(props);
+ }
+ else if (this.partitioning == PartitioningProperty.HASH_PARTITIONED) {
+ return props.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
+ checkCompatiblePartitioningFields(props);
+ }
+ else if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) {
+ return props.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+ props.matchesOrderedPartitioning(this.ordering);
+ }
+ else if (this.partitioning == PartitioningProperty.FORCED_REBALANCED) {
+ return props.getPartitioning() == PartitioningProperty.FORCED_REBALANCED;
+ }
+ else if (this.partitioning == PartitioningProperty.CUSTOM_PARTITIONING) {
+ return props.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
+ checkCompatiblePartitioningFields(props) &&
+ props.getCustomPartitioner().equals(this.customPartitioner);
+
+ }
+ else {
+ throw new CompilerException("Properties matching logic leaves open cases.");
+ }
+ }
+
+ /**
+ * Parametrizes the ship strategy fields of a channel such that the channel produces
+ * the desired global properties.
+ *
+ * @param channel The channel to parametrize.
+ * @param globalDopChange Flag indicating whether the degree of parallelism changes
+ * between sender and receiver.
+ * @param exchangeMode The mode of data exchange (pipelined, always batch,
+ * batch only on shuffle, ...)
+ * @param breakPipeline Indicates whether this data exchange should break
+ * pipelines (unless pipelines are forced).
+ */
+ public void parameterizeChannel(Channel channel, boolean globalDopChange,
+ ExecutionMode exchangeMode, boolean breakPipeline) {
+
+ // safety check. Fully replicated input must be preserved.
+ if (channel.getSource().getGlobalProperties().isFullyReplicated() &&
+ !(this.partitioning == PartitioningProperty.FULL_REPLICATION ||
+ this.partitioning == PartitioningProperty.ANY_DISTRIBUTION))
+ {
+ throw new CompilerException("Fully replicated input must be preserved " +
+ "and may not be converted into another global property.");
+ }
+
+ // if we request nothing, then we need no special strategy. forward, if the number of instances remains
+ // the same, randomly repartition otherwise
+ if (isTrivial() || this.partitioning == PartitioningProperty.ANY_DISTRIBUTION) {
+ ShipStrategyType shipStrategy = globalDopChange ? ShipStrategyType.PARTITION_RANDOM :
+ ShipStrategyType.FORWARD;
+
+ DataExchangeMode em = DataExchangeMode.select(exchangeMode, shipStrategy, breakPipeline);
+ channel.setShipStrategy(shipStrategy, em);
+ return;
+ }
+
+ final GlobalProperties inGlobals = channel.getSource().getGlobalProperties();
+ // if we have no global parallelism change, check if we have already compatible global properties
+ if (!globalDopChange && isMetBy(inGlobals)) {
+ DataExchangeMode em = DataExchangeMode.select(exchangeMode, ShipStrategyType.FORWARD, breakPipeline);
+ channel.setShipStrategy(ShipStrategyType.FORWARD, em);
+ return;
+ }
+
+ // if we fall through the conditions until here, we need to re-establish
+ ShipStrategyType shipType;
+ FieldList partitionKeys;
+ boolean[] sortDirection;
+ Partitioner<?> partitioner;
+
+ switch (this.partitioning) {
+ case FULL_REPLICATION:
+ shipType = ShipStrategyType.BROADCAST;
+ partitionKeys = null;
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case ANY_PARTITIONING:
+ case HASH_PARTITIONED:
+ shipType = ShipStrategyType.PARTITION_HASH;
+ partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case RANGE_PARTITIONED:
+ shipType = ShipStrategyType.PARTITION_RANGE;
+ partitionKeys = this.ordering.getInvolvedIndexes();
+ sortDirection = this.ordering.getFieldSortDirections();
+ partitioner = null;
+
+ if (this.dataDistribution != null) {
+ channel.setDataDistribution(this.dataDistribution);
+ }
+ break;
+
+ case FORCED_REBALANCED:
+ shipType = ShipStrategyType.PARTITION_FORCED_REBALANCE;
+ partitionKeys = null;
+ sortDirection = null;
+ partitioner = null;
+ break;
+
+ case CUSTOM_PARTITIONING:
+ shipType = ShipStrategyType.PARTITION_CUSTOM;
+ partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
+ sortDirection = null;
+ partitioner = this.customPartitioner;
+ break;
+
+ default:
+ throw new CompilerException("Invalid partitioning to create through a data exchange: "
+ + this.partitioning.name());
+ }
+
+ DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
+ channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((partitioning == null) ? 0 : partitioning.ordinal());
+ result = prime * result + ((partitioningFields == null) ? 0 : partitioningFields.hashCode());
+ result = prime * result + ((ordering == null) ? 0 : ordering.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj != null && obj instanceof RequestedGlobalProperties) {
+ RequestedGlobalProperties other = (RequestedGlobalProperties) obj;
+ return (ordering == other.getOrdering() || (ordering != null && ordering.equals(other.getOrdering())))
+ && (partitioning == other.getPartitioning())
+ && (partitioningFields == other.partitioningFields ||
+ (partitioningFields != null && partitioningFields.equals(other.getPartitionedFields())));
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Requested Global Properties [partitioning=" + partitioning +
+ (this.partitioningFields == null ? "" : ", on fields " + this.partitioningFields) +
+ (this.ordering == null ? "" : ", with ordering " + this.ordering) + "]";
+ }
+
+ public RequestedGlobalProperties clone() {
+ try {
+ return (RequestedGlobalProperties) super.clone();
+ } catch (CloneNotSupportedException cnse) {
+ // should never happen, but propagate just in case
+ throw new RuntimeException(cnse);
+ }
+ }
+
+ private boolean checkCompatiblePartitioningFields(GlobalProperties props) {
+ if(this.partitioningFields instanceof FieldList) {
+ // partitioningFields as FieldList requires strict checking!
+ return props.isExactlyPartitionedOnFields((FieldList)this.partitioningFields);
+ } else {
+ return props.isPartitionedOnFields(this.partitioningFields);
+ }
+ }
+}
[23/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
deleted file mode 100644
index 5175d8c..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeForwardTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataexchange;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
-import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This test verifies that the optimizer assigns the correct
- * data exchange mode to a simple forward / shuffle plan.
- *
- * <pre>
- * (source) -> (map) -> (filter) -> (groupBy / reduce)
- * </pre>
- */
-@SuppressWarnings("serial")
-public class DataExchangeModeForwardTest extends CompilerTestBase {
-
-
- @Test
- public void testPipelinedForced() {
- // PIPELINED_FORCED should result in pipelining all the way
- verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testPipelined() {
- // PIPELINED should result in pipelining all the way
- verifySimpleForwardPlan(ExecutionMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testBatch() {
- // BATCH should result in batching the shuffle all the way
- verifySimpleForwardPlan(ExecutionMode.BATCH,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testBatchForced() {
- // BATCH_FORCED should result in batching all the way
- verifySimpleForwardPlan(ExecutionMode.BATCH_FORCED,
- DataExchangeMode.BATCH, DataExchangeMode.BATCH,
- DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
- DataExchangeMode.BATCH, DataExchangeMode.BATCH);
- }
-
- private void verifySimpleForwardPlan(ExecutionMode execMode,
- DataExchangeMode toMap,
- DataExchangeMode toFilter,
- DataExchangeMode toKeyExtractor,
- DataExchangeMode toCombiner,
- DataExchangeMode toReduce,
- DataExchangeMode toSink)
- {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().setExecutionMode(execMode);
-
- DataSet<String> dataSet = env.readTextFile("/never/accessed");
- dataSet
- .map(new MapFunction<String, Integer>() {
- @Override
- public Integer map(String value) {
- return 0;
- }
- })
- .filter(new FilterFunction<Integer>() {
- @Override
- public boolean filter(Integer value) {
- return false;
- }
- })
- .groupBy(new IdentityKeyExtractor<Integer>())
- .reduceGroup(new Top1GroupReducer<Integer>())
- .output(new DiscardingOutputFormat<Integer>());
-
- OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
- SinkPlanNode sinkNode = optPlan.getDataSinks().iterator().next();
-
- SingleInputPlanNode reduceNode = (SingleInputPlanNode) sinkNode.getPredecessor();
- SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getPredecessor();
- SingleInputPlanNode keyExtractorNode = (SingleInputPlanNode) combineNode.getPredecessor();
-
- SingleInputPlanNode filterNode = (SingleInputPlanNode) keyExtractorNode.getPredecessor();
- SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
-
- assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
- assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
- assertEquals(toKeyExtractor, keyExtractorNode.getInput().getDataExchangeMode());
- assertEquals(toCombiner, combineNode.getInput().getDataExchangeMode());
- assertEquals(toReduce, reduceNode.getInput().getDataExchangeMode());
- assertEquals(toSink, sinkNode.getInput().getDataExchangeMode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
deleted file mode 100644
index 6b2691a..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/DataExchangeModeOpenBranchingTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataexchange;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.CompilerTestBase;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.junit.Test;
-
-import java.util.Collection;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This test checks the correct assignment of the DataExchangeMode to
- * connections for programs that branch, but do not re-join the branches.
- *
- * <pre>
- * /---> (filter) -> (sink)
- * /
- * /
- * (source) -> (map) -----------------\
- * \ (join) -> (sink)
- * \ (source) --/
- * \
- * \
- * \-> (sink)
- * </pre>
- */
-@SuppressWarnings({"serial", "unchecked"})
-public class DataExchangeModeOpenBranchingTest extends CompilerTestBase {
-
- @Test
- public void testPipelinedForced() {
- // PIPELINED_FORCED should result in pipelining all the way
- verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testPipelined() {
- // PIPELINED should result in pipelining all the way
- verifyBranchigPlan(ExecutionMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testBatch() {
- // BATCH should result in batching the shuffle all the way
- verifyBranchigPlan(ExecutionMode.BATCH,
- DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
- DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
- DataExchangeMode.PIPELINED);
- }
-
- @Test
- public void testBatchForced() {
- // BATCH_FORCED should result in batching all the way
- verifyBranchigPlan(ExecutionMode.BATCH_FORCED,
- DataExchangeMode.BATCH, DataExchangeMode.BATCH,
- DataExchangeMode.BATCH, DataExchangeMode.BATCH,
- DataExchangeMode.BATCH, DataExchangeMode.BATCH,
- DataExchangeMode.BATCH);
- }
-
- private void verifyBranchigPlan(ExecutionMode execMode,
- DataExchangeMode toMap,
- DataExchangeMode toFilter,
- DataExchangeMode toFilterSink,
- DataExchangeMode toJoin1,
- DataExchangeMode toJoin2,
- DataExchangeMode toJoinSink,
- DataExchangeMode toDirectSink)
- {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().setExecutionMode(execMode);
-
- DataSet<Tuple2<Long, Long>> data = env.generateSequence(1, 100000)
- .map(new MapFunction<Long, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Long value) {
- return new Tuple2<Long, Long>(value, value);
- }
- });
-
- // output 1
- data
- .filter(new FilterFunction<Tuple2<Long, Long>>() {
- @Override
- public boolean filter(Tuple2<Long, Long> value) {
- return false;
- }
- })
- .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("sink1");
-
- // output 2 does a join before a join
- data
- .join(env.fromElements(new Tuple2<Long, Long>(1L, 2L)))
- .where(1)
- .equalTo(0)
- .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>()).name("sink2");
-
- // output 3 is direct
- data
- .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()).name("sink3");
-
- OptimizedPlan optPlan = compileNoStats(env.createProgramPlan());
-
- SinkPlanNode filterSink = findSink(optPlan.getDataSinks(), "sink1");
- SinkPlanNode joinSink = findSink(optPlan.getDataSinks(), "sink2");
- SinkPlanNode directSink = findSink(optPlan.getDataSinks(), "sink3");
-
- SingleInputPlanNode filterNode = (SingleInputPlanNode) filterSink.getPredecessor();
- SingleInputPlanNode mapNode = (SingleInputPlanNode) filterNode.getPredecessor();
-
- DualInputPlanNode joinNode = (DualInputPlanNode) joinSink.getPredecessor();
- assertEquals(mapNode, joinNode.getInput1().getSource());
-
- assertEquals(mapNode, directSink.getPredecessor());
-
- assertEquals(toFilterSink, filterSink.getInput().getDataExchangeMode());
- assertEquals(toJoinSink, joinSink.getInput().getDataExchangeMode());
- assertEquals(toDirectSink, directSink.getInput().getDataExchangeMode());
-
- assertEquals(toMap, mapNode.getInput().getDataExchangeMode());
- assertEquals(toFilter, filterNode.getInput().getDataExchangeMode());
-
- assertEquals(toJoin1, joinNode.getInput1().getDataExchangeMode());
- assertEquals(toJoin2, joinNode.getInput2().getDataExchangeMode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- private SinkPlanNode findSink(Collection<SinkPlanNode> collection, String name) {
- for (SinkPlanNode node : collection) {
- String nodeName = node.getOptimizerNode().getOperator().getName();
- if (nodeName != null && nodeName.equals(name)) {
- return node;
- }
- }
-
- throw new IllegalArgumentException("No node with that name was found.");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
deleted file mode 100644
index 1a14be5..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataexchange/PipelineBreakingTest.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataexchange;
-
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.dag.DataSinkNode;
-import org.apache.flink.optimizer.dag.OptimizerNode;
-import org.apache.flink.optimizer.dag.SingleInputNode;
-import org.apache.flink.optimizer.dag.SinkJoiner;
-import org.apache.flink.optimizer.dag.TwoInputNode;
-import org.apache.flink.optimizer.testfunctions.DummyCoGroupFunction;
-import org.apache.flink.optimizer.testfunctions.DummyFlatJoinFunction;
-import org.apache.flink.optimizer.testfunctions.IdentityFlatMapper;
-import org.apache.flink.optimizer.testfunctions.IdentityKeyExtractor;
-import org.apache.flink.optimizer.testfunctions.SelectOneReducer;
-import org.apache.flink.optimizer.testfunctions.Top1GroupReducer;
-import org.apache.flink.optimizer.traversals.BranchesVisitor;
-import org.apache.flink.optimizer.traversals.GraphCreatingVisitor;
-import org.apache.flink.optimizer.traversals.IdAndEstimatesVisitor;
-import org.junit.Test;
-
-import java.util.Iterator;
-import java.util.List;
-
-import static org.junit.Assert.*;
-
-/**
- * This test checks whether connections are correctly marked as pipelined breaking.
- */
-@SuppressWarnings("serial")
-public class PipelineBreakingTest {
-
- /**
- * Tests that no pipeline breakers are inserted into a simple forward
- * pipeline.
- *
- * <pre>
- * (source) -> (map) -> (filter) -> (groupBy / reduce)
- * </pre>
- */
- @Test
- public void testSimpleForwardPlan() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<String> dataSet = env.readTextFile("/never/accessed");
- dataSet
- .map(new MapFunction<String, Integer>() {
- @Override
- public Integer map(String value) {
- return 0;
- }
- })
- .filter(new FilterFunction<Integer>() {
- @Override
- public boolean filter(Integer value) {
- return false;
- }
- })
- .groupBy(new IdentityKeyExtractor<Integer>())
- .reduceGroup(new Top1GroupReducer<Integer>())
- .output(new DiscardingOutputFormat<Integer>());
-
- DataSinkNode sinkNode = convertPlan(env.createProgramPlan()).get(0);
-
- SingleInputNode reduceNode = (SingleInputNode) sinkNode.getPredecessorNode();
- SingleInputNode keyExtractorNode = (SingleInputNode) reduceNode.getPredecessorNode();
-
- SingleInputNode filterNode = (SingleInputNode) keyExtractorNode.getPredecessorNode();
- SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode();
-
- assertFalse(sinkNode.getInputConnection().isBreakingPipeline());
- assertFalse(reduceNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(keyExtractorNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(filterNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- /**
- * Tests that branching plans, where the branches are not re-joined,
- * do not place pipeline breakers.
- *
- * <pre>
- * /---> (filter) -> (sink)
- * /
- * /
- * (source) -> (map) -----------------\
- * \ (join) -> (sink)
- * \ (source) --/
- * \
- * \
- * \-> (sink)
- * </pre>
- */
- @Test
- public void testBranchingPlanNotReJoined() {
- try {
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Integer> data = env.readTextFile("/never/accessed")
- .map(new MapFunction<String, Integer>() {
- @Override
- public Integer map(String value) {
- return 0;
- }
- });
-
- // output 1
- data
- .filter(new FilterFunction<Integer>() {
- @Override
- public boolean filter(Integer value) {
- return false;
- }
- })
- .output(new DiscardingOutputFormat<Integer>());
-
- // output 2 does a join before a join
- data
- .join(env.fromElements(1, 2, 3, 4))
- .where(new IdentityKeyExtractor<Integer>())
- .equalTo(new IdentityKeyExtractor<Integer>())
- .output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
-
- // output 3 is direct
- data
- .output(new DiscardingOutputFormat<Integer>());
-
- List<DataSinkNode> sinks = convertPlan(env.createProgramPlan());
-
- // gather the optimizer DAG nodes
-
- DataSinkNode sinkAfterFilter = sinks.get(0);
- DataSinkNode sinkAfterJoin = sinks.get(1);
- DataSinkNode sinkDirect = sinks.get(2);
-
- SingleInputNode filterNode = (SingleInputNode) sinkAfterFilter.getPredecessorNode();
- SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode();
-
- TwoInputNode joinNode = (TwoInputNode) sinkAfterJoin.getPredecessorNode();
- SingleInputNode joinInput = (SingleInputNode) joinNode.getSecondPredecessorNode();
-
- // verify the non-pipeline breaking status
-
- assertFalse(sinkAfterFilter.getInputConnection().isBreakingPipeline());
- assertFalse(sinkAfterJoin.getInputConnection().isBreakingPipeline());
- assertFalse(sinkDirect.getInputConnection().isBreakingPipeline());
-
- assertFalse(filterNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
-
- assertFalse(joinNode.getFirstIncomingConnection().isBreakingPipeline());
- assertFalse(joinNode.getSecondIncomingConnection().isBreakingPipeline());
- assertFalse(joinInput.getIncomingConnection().isBreakingPipeline());
-
- // some other sanity checks on the plan construction (cannot hurt)
-
- assertEquals(mapNode, ((SingleInputNode) joinNode.getFirstPredecessorNode()).getPredecessorNode());
- assertEquals(mapNode, sinkDirect.getPredecessorNode());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- /**
- * Tests that branches that are re-joined have place pipeline breakers.
- *
- * <pre>
- * /-> (sink)
- * /
- * /-> (reduce) -+ /-> (flatmap) -> (sink)
- * / \ /
- * (source) -> (map) - (join) -+-----\
- * \ / \
- * \-> (filter) -+ \
- * \ (co group) -> (sink)
- * \ /
- * \-> (reduce) - /
- * </pre>
- */
- @Test
- public void testReJoinedBranches() {
- try {
- // build a test program
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple2<Long, Long>> data = env.fromElements(33L, 44L)
- .map(new MapFunction<Long, Tuple2<Long, Long>>() {
- @Override
- public Tuple2<Long, Long> map(Long value) {
- return new Tuple2<Long, Long>(value, value);
- }
- });
-
- DataSet<Tuple2<Long, Long>> reduced = data.groupBy(0).reduce(new SelectOneReducer<Tuple2<Long, Long>>());
- reduced.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
-
- DataSet<Tuple2<Long, Long>> filtered = data.filter(new FilterFunction<Tuple2<Long, Long>>() {
- @Override
- public boolean filter(Tuple2<Long, Long> value) throws Exception {
- return false;
- }
- });
-
- DataSet<Tuple2<Long, Long>> joined = reduced.join(filtered)
- .where(1).equalTo(1)
- .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
-
- joined.flatMap(new IdentityFlatMapper<Tuple2<Long, Long>>())
- .output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
-
- joined.coGroup(filtered.groupBy(1).reduceGroup(new Top1GroupReducer<Tuple2<Long, Long>>()))
- .where(0).equalTo(0)
- .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>())
- .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
-
- List<DataSinkNode> sinks = convertPlan(env.createProgramPlan());
-
- // gather the optimizer DAG nodes
-
- DataSinkNode sinkAfterReduce = sinks.get(0);
- DataSinkNode sinkAfterFlatMap = sinks.get(1);
- DataSinkNode sinkAfterCoGroup = sinks.get(2);
-
- SingleInputNode reduceNode = (SingleInputNode) sinkAfterReduce.getPredecessorNode();
- SingleInputNode mapNode = (SingleInputNode) reduceNode.getPredecessorNode();
-
- SingleInputNode flatMapNode = (SingleInputNode) sinkAfterFlatMap.getPredecessorNode();
- TwoInputNode joinNode = (TwoInputNode) flatMapNode.getPredecessorNode();
- SingleInputNode filterNode = (SingleInputNode) joinNode.getSecondPredecessorNode();
-
- TwoInputNode coGroupNode = (TwoInputNode) sinkAfterCoGroup.getPredecessorNode();
- SingleInputNode otherReduceNode = (SingleInputNode) coGroupNode.getSecondPredecessorNode();
-
- // test sanity checks (that we constructed the DAG correctly)
-
- assertEquals(reduceNode, joinNode.getFirstPredecessorNode());
- assertEquals(mapNode, filterNode.getPredecessorNode());
- assertEquals(joinNode, coGroupNode.getFirstPredecessorNode());
- assertEquals(filterNode, otherReduceNode.getPredecessorNode());
-
- // verify the pipeline breaking status
-
- assertFalse(sinkAfterReduce.getInputConnection().isBreakingPipeline());
- assertFalse(sinkAfterFlatMap.getInputConnection().isBreakingPipeline());
- assertFalse(sinkAfterCoGroup.getInputConnection().isBreakingPipeline());
-
- assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(flatMapNode.getIncomingConnection().isBreakingPipeline());
- assertFalse(joinNode.getFirstIncomingConnection().isBreakingPipeline());
- assertFalse(coGroupNode.getFirstIncomingConnection().isBreakingPipeline());
- assertFalse(coGroupNode.getSecondIncomingConnection().isBreakingPipeline());
-
- // these should be pipeline breakers
- assertTrue(reduceNode.getIncomingConnection().isBreakingPipeline());
- assertTrue(filterNode.getIncomingConnection().isBreakingPipeline());
- assertTrue(otherReduceNode.getIncomingConnection().isBreakingPipeline());
- assertTrue(joinNode.getSecondIncomingConnection().isBreakingPipeline());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- private static List<DataSinkNode> convertPlan(Plan p) {
- GraphCreatingVisitor dagCreator =
- new GraphCreatingVisitor(17, p.getExecutionConfig().getExecutionMode());
-
- // create the DAG
- p.accept(dagCreator);
- List<DataSinkNode> sinks = dagCreator.getSinks();
-
- // build a single root and run the branch tracking logic
- OptimizerNode rootNode;
- if (sinks.size() == 1) {
- rootNode = sinks.get(0);
- }
- else {
- Iterator<DataSinkNode> iter = sinks.iterator();
- rootNode = iter.next();
-
- while (iter.hasNext()) {
- rootNode = new SinkJoiner(rootNode, iter.next());
- }
- }
- rootNode.accept(new IdAndEstimatesVisitor(null));
- rootNode.accept(new BranchesVisitor());
-
- return sinks;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
deleted file mode 100644
index 3e32905..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesFilteringTest.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.functions.SemanticPropUtil;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.StringValue;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Set;
-
-public class GlobalPropertiesFilteringTest {
-
- private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
- new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
- );
-
- @Test
- public void testAllErased1() {
-
- SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1));
- gprops.addUniqueFieldCombination(new FieldSet(3, 4));
- gprops.addUniqueFieldCombination(new FieldSet(5, 6));
-
- GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- assertNull(result.getPartitioningOrdering());
- assertNull(result.getUniqueFieldCombination());
- }
-
- @Test
- public void testAllErased2() {
-
- SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[]{"2"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1));
- gprops.addUniqueFieldCombination(new FieldSet(3, 4));
- gprops.addUniqueFieldCombination(new FieldSet(5, 6));
-
- GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- assertNull(result.getPartitioningOrdering());
- assertNull(result.getUniqueFieldCombination());
- }
-
- @Test
- public void testHashPartitioningPreserved1() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertTrue(pFields.contains(0));
- assertTrue(pFields.contains(1));
- assertTrue(pFields.contains(4));
- }
-
- @Test
- public void testHashPartitioningPreserved2() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertTrue(pFields.contains(1));
- assertTrue(pFields.contains(2));
- assertTrue(pFields.contains(3));
- }
-
- @Test
- public void testHashPartitioningErased() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- }
-
- @Test
- public void testAnyPartitioningPreserved1() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setAnyPartitioning(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertTrue(pFields.contains(0));
- assertTrue(pFields.contains(1));
- assertTrue(pFields.contains(4));
- }
-
- @Test
- public void testAnyPartitioningPreserved2() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setAnyPartitioning(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertTrue(pFields.contains(1));
- assertTrue(pFields.contains(2));
- assertTrue(pFields.contains(3));
- }
-
- @Test
- public void testAnyPartitioningErased() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setAnyPartitioning(new FieldList(0, 1, 4));
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- }
-
- @Test
- public void testCustomPartitioningPreserved1() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
- gprops.setCustomPartitioned(new FieldList(0, 4), myP);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(2, pFields.size());
- assertTrue(pFields.contains(0));
- assertTrue(pFields.contains(4));
- assertEquals(myP, result.getCustomPartitioner());
- }
-
- @Test
- public void testCustomPartitioningPreserved2() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
- gprops.setCustomPartitioned(new FieldList(0, 4), myP);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(2, pFields.size());
- assertTrue(pFields.contains(1));
- assertTrue(pFields.contains(3));
- assertEquals(myP, result.getCustomPartitioner());
- }
-
- @Test
- public void testCustomPartitioningErased() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
- gprops.setCustomPartitioned(new FieldList(0, 4), myP);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- assertNull(result.getCustomPartitioner());
- }
-
- @Test
- public void testRangePartitioningPreserved1() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;2;5"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- o.appendOrdering(2, StringValue.class, Order.ASCENDING);
- GlobalProperties gprops = new GlobalProperties();
- gprops.setRangePartitioned(o);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertEquals(1, pFields.get(0).intValue());
- assertEquals(5, pFields.get(1).intValue());
- assertEquals(2, pFields.get(2).intValue());
- Ordering pOrder = result.getPartitioningOrdering();
- assertEquals(3, pOrder.getNumberOfFields());
- assertEquals(1, pOrder.getFieldNumber(0).intValue());
- assertEquals(5, pOrder.getFieldNumber(1).intValue());
- assertEquals(2, pOrder.getFieldNumber(2).intValue());
- assertEquals(Order.ASCENDING, pOrder.getOrder(0));
- assertEquals(Order.DESCENDING, pOrder.getOrder(1));
- assertEquals(Order.ASCENDING, pOrder.getOrder(2));
- assertEquals(IntValue.class, pOrder.getType(0));
- assertEquals(LongValue.class, pOrder.getType(1));
- assertEquals(StringValue.class, pOrder.getType(2));
- }
-
- @Test
- public void testRangePartitioningPreserved2() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1->3; 2->0; 5->1"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- o.appendOrdering(2, StringValue.class, Order.ASCENDING);
- GlobalProperties gprops = new GlobalProperties();
- gprops.setRangePartitioned(o);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
- FieldList pFields = result.getPartitioningFields();
- assertEquals(3, pFields.size());
- assertEquals(3, pFields.get(0).intValue());
- assertEquals(1, pFields.get(1).intValue());
- assertEquals(0, pFields.get(2).intValue());
- Ordering pOrder = result.getPartitioningOrdering();
- assertEquals(3, pOrder.getNumberOfFields());
- assertEquals(3, pOrder.getFieldNumber(0).intValue());
- assertEquals(1, pOrder.getFieldNumber(1).intValue());
- assertEquals(0, pOrder.getFieldNumber(2).intValue());
- assertEquals(Order.ASCENDING, pOrder.getOrder(0));
- assertEquals(Order.DESCENDING, pOrder.getOrder(1));
- assertEquals(Order.ASCENDING, pOrder.getOrder(2));
- assertEquals(IntValue.class, pOrder.getType(0));
- assertEquals(LongValue.class, pOrder.getType(1));
- assertEquals(StringValue.class, pOrder.getType(2));
- }
-
- @Test
- public void testRangePartitioningErased() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(1, IntValue.class, Order.ASCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- o.appendOrdering(2, StringValue.class, Order.ASCENDING);
- GlobalProperties gprops = new GlobalProperties();
- gprops.setRangePartitioned(o);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
- assertNull(result.getPartitioningOrdering());
- assertNull(result.getPartitioningFields());
- }
-
- @Test
- public void testRebalancingPreserved() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setForcedRebalanced();
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
-
- assertEquals(PartitioningProperty.FORCED_REBALANCED, result.getPartitioning());
- assertNull(result.getPartitioningFields());
- }
-
- @Test
- public void testUniqueFieldGroupsPreserved1() {
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
-
- FieldSet set1 = new FieldSet(0, 1, 2);
- FieldSet set2 = new FieldSet(3, 4);
- FieldSet set3 = new FieldSet(4, 5, 6, 7);
- GlobalProperties gprops = new GlobalProperties();
- gprops.addUniqueFieldCombination(set1);
- gprops.addUniqueFieldCombination(set2);
- gprops.addUniqueFieldCombination(set3);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
- Set<FieldSet> unique = result.getUniqueFieldCombination();
- FieldSet expected1 = new FieldSet(0, 1, 2);
- FieldSet expected2 = new FieldSet(3, 4);
-
- Assert.assertTrue(unique.size() == 2);
- Assert.assertTrue(unique.contains(expected1));
- Assert.assertTrue(unique.contains(expected2));
- }
-
- @Test
- public void testUniqueFieldGroupsPreserved2() {
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->5;1;2;3->6;4"}, null, null, tupleInfo, tupleInfo);
-
- FieldSet set1 = new FieldSet(0, 1, 2);
- FieldSet set2 = new FieldSet(3, 4);
- FieldSet set3 = new FieldSet(4, 5, 6, 7);
- GlobalProperties gprops = new GlobalProperties();
- gprops.addUniqueFieldCombination(set1);
- gprops.addUniqueFieldCombination(set2);
- gprops.addUniqueFieldCombination(set3);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
- Set<FieldSet> unique = result.getUniqueFieldCombination();
- FieldSet expected1 = new FieldSet(1, 2, 5);
- FieldSet expected2 = new FieldSet(4, 6);
-
- Assert.assertTrue(unique.size() == 2);
- Assert.assertTrue(unique.contains(expected1));
- Assert.assertTrue(unique.contains(expected2));
- }
-
- @Test
- public void testUniqueFieldGroupsErased() {
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0; 3; 5; 6; 7"}, null, null, tupleInfo, tupleInfo);
-
- FieldSet set1 = new FieldSet(0, 1, 2);
- FieldSet set2 = new FieldSet(3, 4);
- FieldSet set3 = new FieldSet(4, 5, 6, 7);
- GlobalProperties gprops = new GlobalProperties();
- gprops.addUniqueFieldCombination(set1);
- gprops.addUniqueFieldCombination(set2);
- gprops.addUniqueFieldCombination(set3);
-
- GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
- Assert.assertNull(result.getUniqueFieldCombination());
- }
-
- @Test(expected = IndexOutOfBoundsException.class)
- public void testInvalidInputIndex() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- GlobalProperties gprops = new GlobalProperties();
- gprops.setHashPartitioned(new FieldList(0, 1));
-
- gprops.filterBySemanticProperties(sprops, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
deleted file mode 100644
index 52826d6..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesMatchingTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.junit.Test;
-
-public class GlobalPropertiesMatchingTest {
-
- @Test
- public void testMatchingAnyPartitioning() {
- try {
-
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setAnyPartitioning(new FieldSet(6, 2));
-
- // match any partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setAnyPartitioning(new FieldList(2, 6));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setAnyPartitioning(new FieldList(6, 2));
- assertTrue(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setAnyPartitioning(new FieldList(6, 2, 4));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setAnyPartitioning(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp4));
-
- GlobalProperties gp5 = new GlobalProperties();
- gp5.setAnyPartitioning(new FieldList(2));
- assertTrue(req.isMetBy(gp5));
- }
-
- // match hash partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setHashPartitioned(new FieldList(2, 6));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setHashPartitioned(new FieldList(6, 2));
- assertTrue(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setHashPartitioned(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp3));
- }
-
- // match range partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
- assertTrue(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
- assertTrue(req.isMetBy(gp4));
- }
-
- // match custom partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setCustomPartitioned(new FieldList(2, 6), new MockPartitioner());
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setCustomPartitioned(new FieldList(6, 2), new MockPartitioner());
- assertTrue(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setCustomPartitioned(new FieldList(6, 1), new MockPartitioner());
- assertFalse(req.isMetBy(gp3));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testMatchingCustomPartitioning() {
- try {
- final Partitioner<Tuple2<Long, Integer>> partitioner = new MockPartitioner();
-
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setCustomPartitioned(new FieldSet(6, 2), partitioner);
-
- // match custom partitionings
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setCustomPartitioned(new FieldList(2, 6), partitioner);
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setCustomPartitioned(new FieldList(6, 2), partitioner);
- assertTrue(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setCustomPartitioned(new FieldList(6, 2), new MockPartitioner());
- assertFalse(req.isMetBy(gp3));
- }
-
- // cannot match other types of partitionings
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setAnyPartitioning(new FieldList(6, 2));
- assertFalse(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setHashPartitioned(new FieldList(6, 2));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp3));
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testStrictlyMatchingAnyPartitioning() {
-
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setAnyPartitioning(new FieldList(6, 2));
-
- // match any partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setAnyPartitioning(new FieldList(6, 2));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setAnyPartitioning(new FieldList(2, 6));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setAnyPartitioning(new FieldList(6, 2, 3));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp3.setAnyPartitioning(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp5 = new GlobalProperties();
- gp4.setAnyPartitioning(new FieldList(2));
- assertFalse(req.isMetBy(gp4));
- }
-
- // match hash partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setHashPartitioned(new FieldList(6, 2));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setHashPartitioned(new FieldList(2, 6));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setHashPartitioned(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp3));
- }
-
- // match range partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
- assertFalse(req.isMetBy(gp4));
- }
-
- }
-
- @Test
- public void testStrictlyMatchingHashPartitioning() {
-
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setHashPartitioned(new FieldList(6, 2));
-
- // match any partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setAnyPartitioning(new FieldList(6, 2));
- assertFalse(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setAnyPartitioning(new FieldList(2, 6));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setAnyPartitioning(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setAnyPartitioning(new FieldList(2));
- assertFalse(req.isMetBy(gp4));
- }
-
- // match hash partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setHashPartitioned(new FieldList(6, 2));
- assertTrue(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setHashPartitioned(new FieldList(2, 6));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setHashPartitioned(new FieldList(6, 1));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setHashPartitioned(new FieldList(6, 2, 0));
- assertFalse(req.isMetBy(gp4));
- }
-
- // match range partitioning
- {
- GlobalProperties gp1 = new GlobalProperties();
- gp1.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(2, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp1));
-
- GlobalProperties gp2 = new GlobalProperties();
- gp2.setRangePartitioned(new Ordering(2, null, Order.DESCENDING).appendOrdering(6, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp2));
-
- GlobalProperties gp3 = new GlobalProperties();
- gp3.setRangePartitioned(new Ordering(6, null, Order.DESCENDING).appendOrdering(1, null, Order.ASCENDING));
- assertFalse(req.isMetBy(gp3));
-
- GlobalProperties gp4 = new GlobalProperties();
- gp4.setRangePartitioned(new Ordering(6, null, Order.DESCENDING));
- assertFalse(req.isMetBy(gp4));
- }
-
- }
-
- // --------------------------------------------------------------------------------------------
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
deleted file mode 100644
index 0868720..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/GlobalPropertiesPushdownTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.operators.SemanticProperties;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.junit.Test;
-
-public class GlobalPropertiesPushdownTest {
-
- @Test
- public void testAnyPartitioningPushedDown() {
- try {
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setAnyPartitioning(new FieldSet(3, 1));
-
- RequestedGlobalProperties preserved = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
- assertEquals(PartitioningProperty.ANY_PARTITIONING, preserved.getPartitioning());
- assertTrue(preserved.getPartitionedFields().isValidSubset(new FieldSet(1, 3)));
-
- RequestedGlobalProperties nonPreserved = req.filterBySemanticProperties(getNonePreservingSemProps(), 0);
- assertTrue(nonPreserved == null || nonPreserved.isTrivial());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testHashPartitioningPushedDown() {
- try {
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setHashPartitioned(new FieldSet(3, 1));
-
- RequestedGlobalProperties preserved = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
- assertEquals(PartitioningProperty.HASH_PARTITIONED, preserved.getPartitioning());
- assertTrue(preserved.getPartitionedFields().isValidSubset(new FieldSet(1, 3)));
-
- RequestedGlobalProperties nonPreserved = req.filterBySemanticProperties(getNonePreservingSemProps(), 0);
- assertTrue(nonPreserved == null || nonPreserved.isTrivial());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testCustomPartitioningNotPushedDown() {
- try {
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setCustomPartitioned(new FieldSet(3, 1), new MockPartitioner());
-
- RequestedGlobalProperties pushedDown = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
- assertTrue(pushedDown == null || pushedDown.isTrivial());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void testForcedReblancingNotPushedDown() {
- try {
- RequestedGlobalProperties req = new RequestedGlobalProperties();
- req.setForceRebalancing();
-
- RequestedGlobalProperties pushedDown = req.filterBySemanticProperties(getAllPreservingSemProps(), 0);
- assertTrue(pushedDown == null || pushedDown.isTrivial());
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private static SemanticProperties getAllPreservingSemProps() {
- return new SingleInputSemanticProperties.AllFieldsForwardedProperties();
- }
-
- private static SemanticProperties getNonePreservingSemProps() {
- return new SingleInputSemanticProperties();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
deleted file mode 100644
index 1ff62ed..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/LocalPropertiesFilteringTest.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import static org.junit.Assert.*;
-
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
-import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.java.functions.SemanticPropUtil;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.types.IntValue;
-import org.apache.flink.types.LongValue;
-import org.apache.flink.types.StringValue;
-import org.junit.Test;
-
-public class LocalPropertiesFilteringTest {
-
- private TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>> tupleInfo =
- new TupleTypeInfo<Tuple8<Integer, Integer, Integer, Integer, Integer, Integer, Integer, Integer>>(
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO
- );
-
- @Test
- public void testAllErased1() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
- lProps = lProps.addUniqueFields(new FieldSet(5,6));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testAllErased2() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"5"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testGroupingPreserved1() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNotNull(filtered.getGroupedFields());
- assertEquals(3, filtered.getGroupedFields().size());
- assertTrue(filtered.getGroupedFields().contains(0));
- assertTrue(filtered.getGroupedFields().contains(2));
- assertTrue(filtered.getGroupedFields().contains(3));
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testGroupingPreserved2() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0;3->7"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNotNull(filtered.getGroupedFields());
- assertEquals(3, filtered.getGroupedFields().size());
- assertTrue(filtered.getGroupedFields().contains(4));
- assertTrue(filtered.getGroupedFields().contains(0));
- assertTrue(filtered.getGroupedFields().contains(7));
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testGroupingErased() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testSortingPreserved1() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(2, IntValue.class, Order.ASCENDING);
- o.appendOrdering(0, StringValue.class, Order.DESCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- LocalProperties lProps = LocalProperties.forOrdering(o);
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldList gFields = filtered.getGroupedFields();
- Ordering order = filtered.getOrdering();
-
- assertNotNull(gFields);
- assertEquals(3, gFields.size());
- assertTrue(gFields.contains(0));
- assertTrue(gFields.contains(2));
- assertTrue(gFields.contains(5));
- assertNotNull(order);
- assertEquals(3, order.getNumberOfFields());
- assertEquals(2, order.getFieldNumber(0).intValue());
- assertEquals(0, order.getFieldNumber(1).intValue());
- assertEquals(5, order.getFieldNumber(2).intValue());
- assertEquals(Order.ASCENDING, order.getOrder(0));
- assertEquals(Order.DESCENDING, order.getOrder(1));
- assertEquals(Order.DESCENDING, order.getOrder(2));
- assertEquals(IntValue.class, order.getType(0));
- assertEquals(StringValue.class, order.getType(1));
- assertEquals(LongValue.class, order.getType(2));
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testSortingPreserved2() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(2, IntValue.class, Order.ASCENDING);
- o.appendOrdering(0, StringValue.class, Order.DESCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- LocalProperties lProps = LocalProperties.forOrdering(o);
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldList gFields = filtered.getGroupedFields();
- Ordering order = filtered.getOrdering();
-
- assertNotNull(gFields);
- assertEquals(3, gFields.size());
- assertTrue(gFields.contains(3));
- assertTrue(gFields.contains(7));
- assertTrue(gFields.contains(1));
- assertNotNull(order);
- assertEquals(3, order.getNumberOfFields());
- assertEquals(7, order.getFieldNumber(0).intValue());
- assertEquals(3, order.getFieldNumber(1).intValue());
- assertEquals(1, order.getFieldNumber(2).intValue());
- assertEquals(Order.ASCENDING, order.getOrder(0));
- assertEquals(Order.DESCENDING, order.getOrder(1));
- assertEquals(Order.DESCENDING, order.getOrder(2));
- assertEquals(IntValue.class, order.getType(0));
- assertEquals(StringValue.class, order.getType(1));
- assertEquals(LongValue.class, order.getType(2));
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testSortingPreserved3() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(2, IntValue.class, Order.ASCENDING);
- o.appendOrdering(0, StringValue.class, Order.DESCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- LocalProperties lProps = LocalProperties.forOrdering(o);
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldList gFields = filtered.getGroupedFields();
- Ordering order = filtered.getOrdering();
-
- assertNotNull(gFields);
- assertEquals(2, gFields.size());
- assertTrue(gFields.contains(0));
- assertTrue(gFields.contains(2));
- assertNotNull(order);
- assertEquals(2, order.getNumberOfFields());
- assertEquals(2, order.getFieldNumber(0).intValue());
- assertEquals(0, order.getFieldNumber(1).intValue());
- assertEquals(Order.ASCENDING, order.getOrder(0));
- assertEquals(Order.DESCENDING, order.getOrder(1));
- assertEquals(IntValue.class, order.getType(0));
- assertEquals(StringValue.class, order.getType(1));
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testSortingPreserved4() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(2, IntValue.class, Order.ASCENDING);
- o.appendOrdering(0, StringValue.class, Order.DESCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- LocalProperties lProps = LocalProperties.forOrdering(o);
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldList gFields = filtered.getGroupedFields();
- Ordering order = filtered.getOrdering();
-
- assertNotNull(gFields);
- assertEquals(1, gFields.size());
- assertTrue(gFields.contains(7));
- assertNotNull(order);
- assertEquals(1, order.getNumberOfFields());
- assertEquals(7, order.getFieldNumber(0).intValue());
- assertEquals(Order.ASCENDING, order.getOrder(0));
- assertEquals(IntValue.class, order.getType(0));
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testSortingErased() {
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);
-
- Ordering o = new Ordering();
- o.appendOrdering(2, IntValue.class, Order.ASCENDING);
- o.appendOrdering(0, StringValue.class, Order.DESCENDING);
- o.appendOrdering(5, LongValue.class, Order.DESCENDING);
- LocalProperties lProps = LocalProperties.forOrdering(o);
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldList gFields = filtered.getGroupedFields();
- Ordering order = filtered.getOrdering();
-
- assertNull(gFields);
- assertNull(order);
- assertNull(filtered.getUniqueFields());
- }
-
- @Test
- public void testUniqueFieldsPreserved1() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = new LocalProperties();
- lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
- lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldSet expected1 = new FieldSet(0,1,2);
- FieldSet expected2 = new FieldSet(3,4);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNotNull(filtered.getUniqueFields());
- assertEquals(2, filtered.getUniqueFields().size());
- assertTrue(filtered.getUniqueFields().contains(expected1));
- assertTrue(filtered.getUniqueFields().contains(expected2));
- }
-
- @Test
- public void testUniqueFieldsPreserved2() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = LocalProperties.forGrouping(new FieldList(1,2));
- lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
- lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldSet expected1 = new FieldSet(0,1,2);
- FieldSet expected2 = new FieldSet(3,4);
-
- assertNull(filtered.getOrdering());
- assertNotNull(filtered.getGroupedFields());
- assertEquals(2, filtered.getGroupedFields().size());
- assertTrue(filtered.getGroupedFields().contains(1));
- assertTrue(filtered.getGroupedFields().contains(2));
- assertNotNull(filtered.getUniqueFields());
- assertEquals(2, filtered.getUniqueFields().size());
- assertTrue(filtered.getUniqueFields().contains(expected1));
- assertTrue(filtered.getUniqueFields().contains(expected2));
- }
-
- @Test
- public void testUniqueFieldsPreserved3() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->7;1->6;2->5;3->4;4->3"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = new LocalProperties();
- lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
- lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
- FieldSet expected1 = new FieldSet(5,6,7);
- FieldSet expected2 = new FieldSet(3,4);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNotNull(filtered.getUniqueFields());
- assertEquals(2, filtered.getUniqueFields().size());
- assertTrue(filtered.getUniqueFields().contains(expected1));
- assertTrue(filtered.getUniqueFields().contains(expected2));
- }
-
- @Test
- public void testUniqueFieldsErased() {
-
- SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lProps = new LocalProperties();
- lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
- lProps = lProps.addUniqueFields(new FieldSet(3,4));
- lProps = lProps.addUniqueFields(new FieldSet(4,5,6));
-
- LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
-
- assertNull(filtered.getGroupedFields());
- assertNull(filtered.getOrdering());
- assertNull(filtered.getUniqueFields());
- }
-
- @Test(expected = IndexOutOfBoundsException.class)
- public void testInvalidInputIndex() {
-
- SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
- SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);
-
- LocalProperties lprops = LocalProperties.forGrouping(new FieldList(0,1));
-
- lprops.filterBySemanticProperties(sprops, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
deleted file mode 100644
index 74126f8..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockDistribution.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.types.Key;
-
-import java.io.IOException;
-
-@SuppressWarnings("serial")
-public class MockDistribution implements DataDistribution {
-
- @Override
- public Key<?>[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
- return new Key<?>[0];
- }
-
- @Override
- public int getNumberOfFields() {
- return 0;
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
-
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java b/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
deleted file mode 100644
index 2b2ab14..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/optimizer/dataproperties/MockPartitioner.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.optimizer.dataproperties;
-
-import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.java.tuple.Tuple2;
-
-class MockPartitioner implements Partitioner<Tuple2<Long, Integer>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public int partition(Tuple2<Long, Integer> key, int numPartitions) {
- return 0;
- }
-}
\ No newline at end of file
[14/53] [abbrv] flink git commit: [optimizer] Rename optimizer
project to "flink-optimizer" (previously flink-compiler)
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
new file mode 100644
index 0000000..3fabad6
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CrossStreamOuterSecondDescriptor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class CrossStreamOuterSecondDescriptor extends CartesianProductDescriptor {
+
+ public CrossStreamOuterSecondDescriptor() {
+ this(true, true);
+ }
+
+ public CrossStreamOuterSecondDescriptor(boolean allowBroadcastFirst, boolean allowBroadcastSecond) {
+ super(allowBroadcastFirst, allowBroadcastSecond);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.NESTEDLOOP_STREAMED_OUTER_SECOND;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ // uniqueness becomes grouping with streamed nested loops
+ if ((in2.getGroupedFields() == null || in2.getGroupedFields().size() == 0) &&
+ in2.getUniqueFields() != null && in2.getUniqueFields().size() > 0)
+ {
+ return LocalProperties.forGrouping(in2.getUniqueFields().iterator().next().toFieldList());
+ } else {
+ return in2.clearUniqueFieldSets();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
new file mode 100644
index 0000000..81c823f
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FilterDescriptor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class FilterDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.FLAT_MAP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Filter ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setAnyDistribution();
+ return Collections.singletonList(rgp);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
new file mode 100644
index 0000000..b915e45
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/FlatMapDescriptor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class FlatMapDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.FLAT_MAP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "FlatMap ("+node.getOperator().getName()+")", in, DriverStrategy.FLAT_MAP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setAnyDistribution();
+ return Collections.singletonList(rgp);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
new file mode 100644
index 0000000..b648386
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupCombineProperties.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The properties file belonging to the GroupCombineNode. It translates the GroupCombine operation
+ * to the driver strategy SORTED_GROUP_COMBINE and sets the relevant grouping and sorting keys.
+ * @see org.apache.flink.optimizer.dag.GroupCombineNode
+ */
+public final class GroupCombineProperties extends OperatorDescriptorSingle {
+
+ private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
+
+ public GroupCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
+ super(groupKeys);
+
+ // if we have an additional ordering, construct the ordering to have primarily the grouping fields
+
+ this.ordering = new Ordering();
+ for (Integer key : this.keyList) {
+ this.ordering.appendOrdering(key, null, Order.ANY);
+ }
+
+ // and next the additional order fields
+ if (additionalOrderKeys != null) {
+ for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
+ Integer field = additionalOrderKeys.getFieldNumber(i);
+ Order order = additionalOrderKeys.getOrder(i);
+ this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
+ }
+ }
+
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.SORTED_GROUP_COMBINE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ node.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ // sorting key info
+ SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode(
+ node,
+ "GroupCombine (" + node.getOperator().getName() + ")",
+ in, // reuse the combine strategy also used in the group reduce
+ DriverStrategy.SORTED_GROUP_COMBINE, this.keyList);
+
+ // set sorting comparator key info
+ singleInputPlanNode.setDriverKeyInfo(this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections(), 0);
+ // set grouping comparator key info
+ singleInputPlanNode.setDriverKeyInfo(this.keyList, 1);
+
+ return singleInputPlanNode;
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties props = new RequestedGlobalProperties();
+ props.setRandomPartitioning();
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED) {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
new file mode 100644
index 0000000..ebd09f2
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceProperties.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+public final class GroupReduceProperties extends OperatorDescriptorSingle {
+
+ private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
+
+ private final Partitioner<?> customPartitioner;
+
+
+ public GroupReduceProperties(FieldSet keys) {
+ this(keys, null, null);
+ }
+
+ public GroupReduceProperties(FieldSet keys, Ordering additionalOrderKeys) {
+ this(keys, additionalOrderKeys, null);
+ }
+
+ public GroupReduceProperties(FieldSet keys, Partitioner<?> customPartitioner) {
+ this(keys, null, customPartitioner);
+ }
+
+ public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
+ super(groupKeys);
+
+ // if we have an additional ordering, construct the ordering to have primarily the grouping fields
+ if (additionalOrderKeys != null) {
+ this.ordering = new Ordering();
+ for (Integer key : this.keyList) {
+ this.ordering.appendOrdering(key, null, Order.ANY);
+ }
+
+ // and next the additional order fields
+ for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
+ Integer field = additionalOrderKeys.getFieldNumber(i);
+ Order order = additionalOrderKeys.getOrder(i);
+ this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
+ }
+ }
+ else {
+ this.ordering = null;
+ }
+
+ this.customPartitioner = customPartitioner;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.SORTED_GROUP_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties props = new RequestedGlobalProperties();
+
+ if (customPartitioner == null) {
+ props.setAnyPartitioning(this.keys);
+ } else {
+ props.setCustomPartitioned(this.keys, this.customPartitioner);
+ }
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ RequestedLocalProperties props = new RequestedLocalProperties();
+ if (this.ordering == null) {
+ props.setGroupedFields(this.keys);
+ } else {
+ props.setOrdering(this.ordering);
+ }
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
new file mode 100644
index 0000000..c4f47d3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+
+public final class GroupReduceWithCombineProperties extends OperatorDescriptorSingle {
+
+ private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
+
+ private final Partitioner<?> customPartitioner;
+
+
+ public GroupReduceWithCombineProperties(FieldSet groupKeys) {
+ this(groupKeys, null, null);
+ }
+
+ public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) {
+ this(groupKeys, additionalOrderKeys, null);
+ }
+
+ public GroupReduceWithCombineProperties(FieldSet groupKeys, Partitioner<?> customPartitioner) {
+ this(groupKeys, null, customPartitioner);
+ }
+
+ public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
+ super(groupKeys);
+
+ // if we have an additional ordering, construct the ordering to have primarily the grouping fields
+ if (additionalOrderKeys != null) {
+ this.ordering = new Ordering();
+ for (Integer key : this.keyList) {
+ this.ordering.appendOrdering(key, null, Order.ANY);
+ }
+
+ // and next the additional order fields
+ for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
+ Integer field = additionalOrderKeys.getFieldNumber(i);
+ Order order = additionalOrderKeys.getOrder(i);
+ this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
+ }
+ } else {
+ this.ordering = null;
+ }
+
+ this.customPartitioner = customPartitioner;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.SORTED_GROUP_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
+ // adjust a sort (changes grouping, so it must be for this driver to combining sort
+ if (in.getLocalStrategy() == LocalStrategy.SORT) {
+ if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
+ throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
+ }
+ in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
+ in.getLocalStrategySortOrder());
+ }
+ return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", in,
+ DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
+ } else {
+ // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
+ Channel toCombiner = new Channel(in.getSource());
+ toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ // create an input node for combine with same DOP as input node
+ GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator()
+ .getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
+ combiner.setCosts(new Costs(0, 0));
+ combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
+ // set sorting comparator key info
+ combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
+ // set grouping comparator key info
+ combiner.setDriverKeyInfo(this.keyList, 1);
+
+ Channel toReducer = new Channel(combiner);
+ toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
+ in.getShipStrategySortOrder(), in.getDataExchangeMode());
+ toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
+ in.getLocalStrategySortOrder());
+
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
+ toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
+ }
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties props = new RequestedGlobalProperties();
+ if (customPartitioner == null) {
+ props.setAnyPartitioning(this.keys);
+ } else {
+ props.setCustomPartitioned(this.keys, this.customPartitioner);
+ }
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ RequestedLocalProperties props = new RequestedLocalProperties();
+ if (this.ordering == null) {
+ props.setGroupedFields(this.keys);
+ } else {
+ props.setOrdering(this.ordering);
+ }
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
new file mode 100644
index 0000000..fec72a9
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildFirstProperties.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class HashJoinBuildFirstProperties extends AbstractJoinDescriptor {
+
+ public HashJoinBuildFirstProperties(FieldList keys1, FieldList keys2) {
+ super(keys1, keys2);
+ }
+
+ public HashJoinBuildFirstProperties(FieldList keys1, FieldList keys2,
+ boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
+ {
+ super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.HYBRIDHASH_BUILD_FIRST;
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new LocalPropertiesPair(new RequestedLocalProperties(), new RequestedLocalProperties()));
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ DriverStrategy strategy;
+
+ if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
+ // sanity check that the first input is cached and remove that cache
+ if (!in1.getTempMode().isCached()) {
+ throw new CompilerException("No cache at point where static and dynamic parts meet.");
+ }
+ in1.setTempMode(in1.getTempMode().makeNonCached());
+ strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
+ }
+ else {
+ strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
+ }
+ return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return new LocalProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
new file mode 100644
index 0000000..f9d1e6c
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/HashJoinBuildSecondProperties.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+public final class HashJoinBuildSecondProperties extends AbstractJoinDescriptor {
+
+ public HashJoinBuildSecondProperties(FieldList keys1, FieldList keys2) {
+ super(keys1, keys2);
+ }
+
+ public HashJoinBuildSecondProperties(FieldList keys1, FieldList keys2,
+ boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
+ {
+ super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.HYBRIDHASH_BUILD_SECOND;
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new LocalPropertiesPair(
+ new RequestedLocalProperties(), new RequestedLocalProperties()));
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ DriverStrategy strategy;
+
+ if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
+ // sanity check that the first input is cached and remove that cache
+ if (!in2.getTempMode().isCached()) {
+ throw new CompilerException("No cache at point where static and dynamic parts meet.");
+ }
+
+ in2.setTempMode(in2.getTempMode().makeNonCached());
+ strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
+ }
+ else {
+ strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
+ }
+ return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return new LocalProperties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
new file mode 100644
index 0000000..9f14d2a
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapDescriptor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class MapDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.MAP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.MAP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setAnyDistribution();
+ return Collections.singletonList(rgp);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
new file mode 100644
index 0000000..1489097
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/MapPartitionDescriptor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class MapPartitionDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.MAP_PARTITION;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "MapPartition ("+node.getOperator().getName()+")", in, DriverStrategy.MAP_PARTITION);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties rgp = new RequestedGlobalProperties();
+ rgp.setAnyDistribution();
+ return Collections.singletonList(rgp);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
new file mode 100644
index 0000000..7ae35c3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/NoOpDescriptor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+
+public class NoOpDescriptor extends OperatorDescriptorSingle {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.UNARY_NO_OP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "Pipe", in, DriverStrategy.UNARY_NO_OP);
+ }
+
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return gProps;
+ }
+
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
new file mode 100644
index 0000000..c21593e
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+
+/**
+ *
+ */
+public abstract class OperatorDescriptorDual implements AbstractOperatorDescriptor {
+
+ protected final FieldList keys1;
+ protected final FieldList keys2;
+
+ private List<GlobalPropertiesPair> globalProps;
+ private List<LocalPropertiesPair> localProps;
+
+ protected OperatorDescriptorDual() {
+ this(null, null);
+ }
+
+ protected OperatorDescriptorDual(FieldList keys1, FieldList keys2) {
+ this.keys1 = keys1;
+ this.keys2 = keys2;
+ }
+
+ public List<GlobalPropertiesPair> getPossibleGlobalProperties() {
+ if (this.globalProps == null) {
+ this.globalProps = createPossibleGlobalProperties();
+ }
+
+ return this.globalProps;
+ }
+
+ public List<LocalPropertiesPair> getPossibleLocalProperties() {
+ if (this.localProps == null) {
+ this.localProps = createPossibleLocalProperties();
+ }
+
+ return this.localProps;
+ }
+
+ protected abstract List<GlobalPropertiesPair> createPossibleGlobalProperties();
+
+ protected abstract List<LocalPropertiesPair> createPossibleLocalProperties();
+
+ public abstract boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2);
+
+ public abstract boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2);
+
+ public abstract DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node);
+
+ public abstract GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2);
+
+ public abstract LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2);
+
+ protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2) {
+
+ // check number of produced partitioning fields
+ if(fields1.size() != fields2.size()) {
+ return false;
+ } else {
+ return checkEquivalentFieldPositionsInKeyFields(fields1, fields2, fields1.size());
+ }
+ }
+
+ protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2, int numRelevantFields) {
+
+ // check number of produced partitioning fields
+ if(fields1.size() < numRelevantFields || fields2.size() < numRelevantFields) {
+ return false;
+ }
+ else {
+ for(int i=0; i<numRelevantFields; i++) {
+ int pField1 = fields1.get(i);
+ int pField2 = fields2.get(i);
+ // check if position of both produced fields is the same in both requested fields
+ int j;
+ for(j=0; j<this.keys1.size(); j++) {
+ if(this.keys1.get(j) == pField1 && this.keys2.get(j) == pField2) {
+ break;
+ }
+ else if(this.keys1.get(j) != pField1 && this.keys2.get(j) != pField2) {
+ // do nothing
+ }
+ else {
+ return false;
+ }
+ }
+ if(j == this.keys1.size()) {
+ throw new CompilerException("Fields were not found in key fields.");
+ }
+ }
+ }
+ return true;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public static final class GlobalPropertiesPair {
+
+ private final RequestedGlobalProperties props1, props2;
+
+ public GlobalPropertiesPair(RequestedGlobalProperties props1, RequestedGlobalProperties props2) {
+ this.props1 = props1;
+ this.props2 = props2;
+ }
+
+ public RequestedGlobalProperties getProperties1() {
+ return this.props1;
+ }
+
+ public RequestedGlobalProperties getProperties2() {
+ return this.props2;
+ }
+
+ @Override
+ public int hashCode() {
+ return (this.props1 == null ? 0 : this.props1.hashCode()) ^ (this.props2 == null ? 0 : this.props2.hashCode());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == GlobalPropertiesPair.class) {
+ final GlobalPropertiesPair other = (GlobalPropertiesPair) obj;
+
+ return (this.props1 == null ? other.props1 == null : this.props1.equals(other.props1)) &&
+ (this.props2 == null ? other.props2 == null : this.props2.equals(other.props2));
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + this.props1 + " / " + this.props2 + "}";
+ }
+ }
+
+ public static final class LocalPropertiesPair {
+
+ private final RequestedLocalProperties props1, props2;
+
+ public LocalPropertiesPair(RequestedLocalProperties props1, RequestedLocalProperties props2) {
+ this.props1 = props1;
+ this.props2 = props2;
+ }
+
+ public RequestedLocalProperties getProperties1() {
+ return this.props1;
+ }
+
+ public RequestedLocalProperties getProperties2() {
+ return this.props2;
+ }
+
+ @Override
+ public int hashCode() {
+ return (this.props1 == null ? 0 : this.props1.hashCode()) ^ (this.props2 == null ? 0 : this.props2.hashCode());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == LocalPropertiesPair.class) {
+ final LocalPropertiesPair other = (LocalPropertiesPair) obj;
+
+ return (this.props1 == null ? other.props1 == null : this.props1.equals(other.props1)) &&
+ (this.props2 == null ? other.props2 == null : this.props2.equals(other.props2));
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + this.props1 + " / " + this.props2 + "}";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
new file mode 100644
index 0000000..c8be5d4
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorSingle.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+
+/**
+ * Abstract base class for Operator descriptions which instantiates the node and sets the driver
+ * strategy and the sorting and grouping keys. Returns possible local and global properties and
+ * updates them after the operation has been performed.
+ * @see org.apache.flink.compiler.dag.SingleInputNode
+ */
+public abstract class OperatorDescriptorSingle implements AbstractOperatorDescriptor {
+
+ protected final FieldSet keys; // the set of key fields
+ protected final FieldList keyList; // the key fields with ordered field positions
+
+ private List<RequestedGlobalProperties> globalProps;
+ private List<RequestedLocalProperties> localProps;
+
+
+ protected OperatorDescriptorSingle() {
+ this(null);
+ }
+
+ protected OperatorDescriptorSingle(FieldSet keys) {
+ this.keys = keys;
+ this.keyList = keys == null ? null : keys.toFieldList();
+ }
+
+
+ public List<RequestedGlobalProperties> getPossibleGlobalProperties() {
+ if (this.globalProps == null) {
+ this.globalProps = createPossibleGlobalProperties();
+ }
+ return this.globalProps;
+ }
+
+ public List<RequestedLocalProperties> getPossibleLocalProperties() {
+ if (this.localProps == null) {
+ this.localProps = createPossibleLocalProperties();
+ }
+ return this.localProps;
+ }
+
+ /**
+ * Returns a list of global properties that are required by this operator descriptor.
+ *
+ * @return A list of global properties that are required by this operator descriptor.
+ */
+ protected abstract List<RequestedGlobalProperties> createPossibleGlobalProperties();
+
+ /**
+ * Returns a list of local properties that are required by this operator descriptor.
+ *
+ * @return A list of local properties that are required by this operator descriptor.
+ */
+ protected abstract List<RequestedLocalProperties> createPossibleLocalProperties();
+
+ public abstract SingleInputPlanNode instantiate(Channel in, SingleInputNode node);
+
+ /**
+ * Returns the global properties which are present after the operator was applied on the
+ * provided global properties.
+ *
+ * @param in The global properties on which the operator is applied.
+ * @return The global properties which are valid after the operator has been applied.
+ */
+ public abstract GlobalProperties computeGlobalProperties(GlobalProperties in);
+
+ /**
+ * Returns the local properties which are present after the operator was applied on the
+ * provided local properties.
+ *
+ * @param in The local properties on which the operator is applied.
+ * @return The local properties which are valid after the operator has been applied.
+ */
+ public abstract LocalProperties computeLocalProperties(LocalProperties in);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
new file mode 100644
index 0000000..2bde29b
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/PartialGroupProperties.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+public final class PartialGroupProperties extends OperatorDescriptorSingle {
+
+ public PartialGroupProperties(FieldSet keys) {
+ super(keys);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.SORTED_GROUP_COMBINE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ // create in input node for combine with same DOP as input node
+ GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine("+node.getOperator().getName()+")", in,
+ DriverStrategy.SORTED_GROUP_COMBINE);
+ // sorting key info
+ combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
+ // set grouping comparator key info
+ combiner.setDriverKeyInfo(this.keyList, 1);
+
+ return combiner;
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ return Collections.singletonList(new RequestedGlobalProperties());
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ RequestedLocalProperties props = new RequestedLocalProperties();
+ props.setGroupedFields(this.keys);
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
new file mode 100644
index 0000000..5bb51f3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.ReduceNode;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.PartitioningProperty;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.runtime.operators.util.LocalStrategy;
+
+public final class ReduceProperties extends OperatorDescriptorSingle {
+
+ private final Partitioner<?> customPartitioner;
+
+ public ReduceProperties(FieldSet keys) {
+ this(keys, null);
+ }
+
+ public ReduceProperties(FieldSet keys, Partitioner<?> customPartitioner) {
+ super(keys);
+ this.customPartitioner = customPartitioner;
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.SORTED_REDUCE;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
+ (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty()))
+ {
+ return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
+ DriverStrategy.SORTED_REDUCE, this.keyList);
+ }
+ else {
+ // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
+ Channel toCombiner = new Channel(in.getSource());
+ toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
+
+ // create an input node for combine with same DOP as input node
+ ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
+ combinerNode.setDegreeOfParallelism(in.getSource().getParallelism());
+
+ SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
+ "Combine ("+node.getOperator().getName()+")", toCombiner,
+ DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList);
+
+ combiner.setCosts(new Costs(0, 0));
+ combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
+
+ Channel toReducer = new Channel(combiner);
+ toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
+ in.getShipStrategySortOrder(), in.getDataExchangeMode());
+ toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
+
+ return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", toReducer,
+ DriverStrategy.SORTED_REDUCE, this.keyList);
+ }
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties props = new RequestedGlobalProperties();
+ if (customPartitioner == null) {
+ props.setAnyPartitioning(this.keys);
+ } else {
+ props.setCustomPartitioned(this.keys, this.customPartitioner);
+ }
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ RequestedLocalProperties props = new RequestedLocalProperties();
+ props.setGroupedFields(this.keys);
+ return Collections.singletonList(props);
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
+ gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
+ {
+ gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
+ }
+ gProps.clearUniqueFieldCombinations();
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
new file mode 100644
index 0000000..1dcd87d
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SolutionSetDeltaOperator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.dag.SingleInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.SingleInputPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class SolutionSetDeltaOperator extends OperatorDescriptorSingle {
+
+ public SolutionSetDeltaOperator(FieldList partitioningFields) {
+ super(partitioningFields);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.UNARY_NO_OP;
+ }
+
+ @Override
+ public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
+ return new SingleInputPlanNode(node, "SolutionSet Delta", in, DriverStrategy.UNARY_NO_OP);
+ }
+
+ @Override
+ protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
+ RequestedGlobalProperties partProps = new RequestedGlobalProperties();
+ partProps.setHashPartitioned(this.keyList);
+ return Collections.singletonList(partProps);
+ }
+
+ @Override
+ protected List<RequestedLocalProperties> createPossibleLocalProperties() {
+ return Collections.singletonList(new RequestedLocalProperties());
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
+ return gProps;
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties lProps) {
+ return lProps;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
new file mode 100644
index 0000000..356836a
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/SortMergeJoinDescriptor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.util.FieldList;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.util.Utils;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class SortMergeJoinDescriptor extends AbstractJoinDescriptor {
+
+ public SortMergeJoinDescriptor(FieldList keys1, FieldList keys2) {
+ super(keys1, keys2);
+ }
+
+ public SortMergeJoinDescriptor(FieldList keys1, FieldList keys2,
+ boolean broadcastFirstAllowed, boolean broadcastSecondAllowed, boolean repartitionAllowed)
+ {
+ super(keys1, keys2, broadcastFirstAllowed, broadcastSecondAllowed, repartitionAllowed);
+ }
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.MERGE;
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ RequestedLocalProperties sort1 = new RequestedLocalProperties(Utils.createOrdering(this.keys1));
+ RequestedLocalProperties sort2 = new RequestedLocalProperties(Utils.createOrdering(this.keys2));
+ return Collections.singletonList(new LocalPropertiesPair(sort1, sort2));
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2)
+ {
+ int numRelevantFields = this.keys1.size();
+
+ Ordering prod1 = produced1.getOrdering();
+ Ordering prod2 = produced2.getOrdering();
+
+ if (prod1 == null || prod2 == null) {
+ throw new CompilerException("The given properties do not meet this operators requirements.");
+ }
+
+ // check that order of fields is equivalent
+ if (!checkEquivalentFieldPositionsInKeyFields(
+ prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
+ return false;
+ }
+
+ // check that both inputs have the same directions of order
+ for (int i = 0; i < numRelevantFields; i++) {
+ if (prod1.getOrder(i) != prod2.getOrder(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ boolean[] inputOrders = in1.getLocalProperties().getOrdering().getFieldSortDirections();
+
+ if (inputOrders == null || inputOrders.length < this.keys1.size()) {
+ throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a merge operator.");
+ } else if (inputOrders.length > this.keys1.size()) {
+ boolean[] tmp = new boolean[this.keys1.size()];
+ System.arraycopy(inputOrders, 0, tmp, 0, tmp.length);
+ inputOrders = tmp;
+ }
+
+ return new DualInputPlanNode(node, "Join("+node.getOperator().getName()+")", in1, in2, DriverStrategy.MERGE, this.keys1, this.keys2, inputOrders);
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ LocalProperties comb = LocalProperties.combine(in1, in2);
+ return comb.clearUniqueFieldSets();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
new file mode 100644
index 0000000..c42cff2
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/UtilSinkJoinOpDescriptor.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.operators;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.dag.SinkJoiner;
+import org.apache.flink.optimizer.dag.TwoInputNode;
+import org.apache.flink.optimizer.dataproperties.GlobalProperties;
+import org.apache.flink.optimizer.dataproperties.LocalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
+import org.apache.flink.optimizer.dataproperties.RequestedLocalProperties;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.SinkJoinerPlanNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ *
+ */
+public class UtilSinkJoinOpDescriptor extends OperatorDescriptorDual {
+
+ @Override
+ public DriverStrategy getStrategy() {
+ return DriverStrategy.BINARY_NO_OP;
+ }
+
+ @Override
+ protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new GlobalPropertiesPair(
+ new RequestedGlobalProperties(), new RequestedGlobalProperties()));
+ }
+
+ @Override
+ protected List<LocalPropertiesPair> createPossibleLocalProperties() {
+ // all properties are possible
+ return Collections.singletonList(new LocalPropertiesPair(
+ new RequestedLocalProperties(), new RequestedLocalProperties()));
+ }
+
+ @Override
+ public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
+ GlobalProperties produced1, GlobalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2,
+ LocalProperties produced1, LocalProperties produced2) {
+ return true;
+ }
+
+ @Override
+ public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
+ if (node instanceof SinkJoiner) {
+ return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2);
+ } else {
+ throw new CompilerException();
+ }
+ }
+
+ @Override
+ public LocalProperties computeLocalProperties(LocalProperties in1, LocalProperties in2) {
+ return new LocalProperties();
+ }
+
+ @Override
+ public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
+ return GlobalProperties.combine(in1, in2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
new file mode 100644
index 0000000..bf22fb3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BinaryUnionPlanNode.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.plan;
+
+import org.apache.flink.optimizer.dag.BinaryUnionNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+
+/**
+ * A special subclass for the union to make it identifiable.
+ */
+public class BinaryUnionPlanNode extends DualInputPlanNode {
+
+ /**
+ * @param template
+ */
+ public BinaryUnionPlanNode(BinaryUnionNode template, Channel in1, Channel in2) {
+ super(template, "Union", in1, in2, DriverStrategy.UNION);
+ }
+
+ public BinaryUnionPlanNode(BinaryUnionPlanNode toSwapFrom) {
+ super(toSwapFrom.getOptimizerNode(), "Union-With-Cached", toSwapFrom.getInput2(), toSwapFrom.getInput1(),
+ DriverStrategy.UNION_WITH_CACHED);
+
+ this.globalProps = toSwapFrom.globalProps;
+ this.localProps = toSwapFrom.localProps;
+ this.nodeCosts = toSwapFrom.nodeCosts;
+ this.cumulativeCosts = toSwapFrom.cumulativeCosts;
+
+ setParallelism(toSwapFrom.getParallelism());
+ }
+
+ public BinaryUnionNode getOptimizerNode() {
+ return (BinaryUnionNode) this.template;
+ }
+
+ public boolean unionsStaticAndDynamicPath() {
+ return getInput1().isOnDynamicPath() != getInput2().isOnDynamicPath();
+ }
+
+ @Override
+ public int getMemoryConsumerWeight() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/633b0d6a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
new file mode 100644
index 0000000..e79e2f3
--- /dev/null
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/BulkIterationPlanNode.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.optimizer.plan;
+
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE;
+import static org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport.FOUND_SOURCE_AND_DAM;
+
+import java.util.HashMap;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.optimizer.CompilerException;
+import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.BulkIterationNode;
+import org.apache.flink.optimizer.dag.OptimizerNode;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Visitor;
+
+public class BulkIterationPlanNode extends SingleInputPlanNode implements IterationPlanNode {
+
+ private final BulkPartialSolutionPlanNode partialSolutionPlanNode;
+
+ private final PlanNode rootOfStepFunction;
+
+ private PlanNode rootOfTerminationCriterion;
+
+ private TypeSerializerFactory<?> serializerForIterationChannel;
+
+ // --------------------------------------------------------------------------------------------
+
+ public BulkIterationPlanNode(BulkIterationNode template, String nodeName, Channel input,
+ BulkPartialSolutionPlanNode pspn, PlanNode rootOfStepFunction)
+ {
+ super(template, nodeName, input, DriverStrategy.NONE);
+ this.partialSolutionPlanNode = pspn;
+ this.rootOfStepFunction = rootOfStepFunction;
+
+ mergeBranchPlanMaps();
+ }
+
+ public BulkIterationPlanNode(BulkIterationNode template, String nodeName, Channel input,
+ BulkPartialSolutionPlanNode pspn, PlanNode rootOfStepFunction, PlanNode rootOfTerminationCriterion)
+ {
+ this(template, nodeName, input, pspn, rootOfStepFunction);
+ this.rootOfTerminationCriterion = rootOfTerminationCriterion;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public BulkIterationNode getIterationNode() {
+ if (this.template instanceof BulkIterationNode) {
+ return (BulkIterationNode) this.template;
+ } else {
+ throw new RuntimeException();
+ }
+ }
+
+ public BulkPartialSolutionPlanNode getPartialSolutionPlanNode() {
+ return this.partialSolutionPlanNode;
+ }
+
+ public PlanNode getRootOfStepFunction() {
+ return this.rootOfStepFunction;
+ }
+
+ public PlanNode getRootOfTerminationCriterion() {
+ return this.rootOfTerminationCriterion;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ public TypeSerializerFactory<?> getSerializerForIterationChannel() {
+ return serializerForIterationChannel;
+ }
+
+ public void setSerializerForIterationChannel(TypeSerializerFactory<?> serializerForIterationChannel) {
+ this.serializerForIterationChannel = serializerForIterationChannel;
+ }
+
+ public void setCosts(Costs nodeCosts) {
+ // add the costs from the step function
+ nodeCosts.addCosts(this.rootOfStepFunction.getCumulativeCosts());
+
+ // add the costs for the termination criterion, if it exists
+ // the costs are divided at branches, so we can simply add them up
+ if (rootOfTerminationCriterion != null) {
+ nodeCosts.addCosts(this.rootOfTerminationCriterion.getCumulativeCosts());
+ }
+
+ super.setCosts(nodeCosts);
+ }
+
+ public int getMemoryConsumerWeight() {
+ return 1;
+ }
+
+
+ @Override
+ public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) {
+ if (source == this) {
+ return FOUND_SOURCE;
+ }
+
+ SourceAndDamReport fromOutside = super.hasDamOnPathDownTo(source);
+
+ if (fromOutside == FOUND_SOURCE_AND_DAM) {
+ return FOUND_SOURCE_AND_DAM;
+ }
+ else if (fromOutside == FOUND_SOURCE) {
+ // we always have a dam in the back channel
+ return FOUND_SOURCE_AND_DAM;
+ } else {
+ // check the step function for dams
+ return this.rootOfStepFunction.hasDamOnPathDownTo(source);
+ }
+ }
+
+ @Override
+ public void acceptForStepFunction(Visitor<PlanNode> visitor) {
+ this.rootOfStepFunction.accept(visitor);
+
+ if(this.rootOfTerminationCriterion != null) {
+ this.rootOfTerminationCriterion.accept(visitor);
+ }
+ }
+
+ private void mergeBranchPlanMaps() {
+ for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) {
+ OptimizerNode brancher = desc.getBranchingNode();
+
+ if (branchPlan == null) {
+ branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
+ }
+
+ if (!branchPlan.containsKey(brancher)) {
+ PlanNode selectedCandidate = null;
+
+ if (rootOfStepFunction.branchPlan != null) {
+ selectedCandidate = rootOfStepFunction.branchPlan.get(brancher);
+ }
+
+ if (selectedCandidate == null) {
+ throw new CompilerException(
+ "Candidates for a node with open branches are missing information about the selected candidate ");
+ }
+
+ this.branchPlan.put(brancher, selectedCandidate);
+ }
+ }
+ }
+}