You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/08/21 19:36:07 UTC
[4/4] flink git commit: [tests] Move UnionClosedBranchingTest to
optimizer tests
[tests] Move UnionClosedBranchingTest to optimizer tests
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a82bd431
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a82bd431
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a82bd431
Branch: refs/heads/master
Commit: a82bd4311240c3eca1656a05b149d85734fd6196
Parents: c9cfb17
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Aug 21 17:58:50 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Aug 21 19:35:39 2015 +0200
----------------------------------------------------------------------
.../dataexchange/UnionClosedBranchingTest.java | 192 +++++++++++++++++++
.../flink/test/UnionClosedBranchingTest.java | 192 -------------------
2 files changed, 192 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a82bd431/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/UnionClosedBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/UnionClosedBranchingTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/UnionClosedBranchingTest.java
new file mode 100644
index 0000000..b870a91
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/dataexchange/UnionClosedBranchingTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.optimizer.dataexchange;
+
+import org.apache.flink.api.common.ExecutionMode;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.optimizer.plan.Channel;
+import org.apache.flink.optimizer.plan.DualInputPlanNode;
+import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.plan.SourcePlanNode;
+import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.io.network.DataExchangeMode;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
+import org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.apache.flink.runtime.io.network.DataExchangeMode.BATCH;
+import static org.apache.flink.runtime.io.network.DataExchangeMode.PIPELINED;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This tests a fix for FLINK-2540.
+ *
+ * <p> This test is necessary, because {@link NAryUnionPlanNode}s are not directly translated
+ * to runtime tasks by the {@link JobGraphGenerator}. Instead, the network stack unions the
+ * inputs by directly reading from multiple inputs (via {@link UnionInputGate}).
+ *
+ * <pre>
+ * (source)-\ /-\
+ * (union)-+ (join)
+ * (source)-/ \-/
+ * </pre>
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/FLINK-2540">FLINK-2540</a>
+ */
+@RunWith(Parameterized.class)
+@SuppressWarnings({"serial","unchecked"})
+public class UnionClosedBranchingTest extends CompilerTestBase {
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ Collection<Object[]> params = Arrays.asList(new Object[][]{
+ {ExecutionMode.PIPELINED, PIPELINED, BATCH},
+ {ExecutionMode.PIPELINED_FORCED, PIPELINED, PIPELINED},
+ {ExecutionMode.BATCH, BATCH, BATCH},
+ {ExecutionMode.BATCH_FORCED, BATCH, BATCH},
+ });
+
+ // Make sure that changes to ExecutionMode are reflected in this test.
+ assertEquals(ExecutionMode.values().length, params.size());
+
+ return params;
+ }
+
+ private final ExecutionMode executionMode;
+
+ /** Expected {@link DataExchangeMode} from sources to union. */
+ private final DataExchangeMode sourceToUnion;
+
+ /** Expected {@link DataExchangeMode} from union to join. */
+ private final DataExchangeMode unionToJoin;
+
+ public UnionClosedBranchingTest(
+ ExecutionMode executionMode,
+ DataExchangeMode sourceToUnion,
+ DataExchangeMode unionToJoin) {
+
+ this.executionMode = executionMode;
+ this.sourceToUnion = sourceToUnion;
+ this.unionToJoin = unionToJoin;
+ }
+
+ @Test
+ public void testUnionClosedBranchingTest() throws Exception {
+
+ // -----------------------------------------------------------------------------------------
+ // Build test program
+ // -----------------------------------------------------------------------------------------
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.getConfig().setExecutionMode(executionMode);
+ env.setParallelism(4);
+
+ DataSet<Tuple1<Integer>> src1 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
+
+ DataSet<Tuple1<Integer>> src2 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
+
+ DataSet<Tuple1<Integer>> union = src1.union(src2);
+
+ DataSet<Tuple2<Integer, Integer>> join = union
+ .join(union).where(0).equalTo(0)
+ .projectFirst(0).projectSecond(0);
+
+ join.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
+
+ // -----------------------------------------------------------------------------------------
+ // Verify optimized plan
+ // -----------------------------------------------------------------------------------------
+
+ OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());
+
+ SinkPlanNode sinkNode = optimizedPlan.getDataSinks().iterator().next();
+
+ DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
+
+ // Verify that the compiler correctly sets the expected data exchange modes.
+ for (Channel channel : joinNode.getInputs()) {
+ assertEquals("Unexpected data exchange mode between union and join node.",
+ unionToJoin, channel.getDataExchangeMode());
+ }
+
+ for (SourcePlanNode src : optimizedPlan.getDataSources()) {
+ for (Channel channel : src.getOutgoingChannels()) {
+ assertEquals("Unexpected data exchange mode between source and union node.",
+ sourceToUnion, channel.getDataExchangeMode());
+ }
+ }
+
+ // -----------------------------------------------------------------------------------------
+ // Verify generated JobGraph
+ // -----------------------------------------------------------------------------------------
+
+ JobGraphGenerator jgg = new JobGraphGenerator();
+ JobGraph jobGraph = jgg.compileJobGraph(optimizedPlan);
+
+ List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
+
+ // Sanity check for the test setup
+ assertEquals("Unexpected number of vertices created.", 4, vertices.size());
+
+ // Verify all sources
+ JobVertex[] sources = new JobVertex[]{vertices.get(0), vertices.get(1)};
+
+ for (JobVertex src : sources) {
+ // Sanity check
+ assertTrue("Unexpected vertex type. Test setup is broken.", src.isInputVertex());
+
+ // The union is not translated to an extra union task, but the join uses a union
+ // input gate to read multiple inputs. The source create a single result per consumer.
+ assertEquals("Unexpected number of created results.", 2,
+ src.getNumberOfProducedIntermediateDataSets());
+
+ for (IntermediateDataSet dataSet : src.getProducedDataSets()) {
+ ResultPartitionType dsType = dataSet.getResultType();
+
+ // The result type is determined by the channel between the union and the join node
+ // and *not* the channel between source and union.
+ if (unionToJoin.equals(BATCH)) {
+ assertTrue("Expected batch exchange, but result type is " + dsType + ".",
+ dsType.isBlocking());
+ } else {
+ assertFalse("Expected non-batch exchange, but result type is " + dsType + ".",
+ dsType.isBlocking());
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a82bd431/flink-tests/src/test/java/org/apache/flink/test/UnionClosedBranchingTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/UnionClosedBranchingTest.java b/flink-tests/src/test/java/org/apache/flink/test/UnionClosedBranchingTest.java
deleted file mode 100644
index f7ea911..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/UnionClosedBranchingTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.test;
-
-import org.apache.flink.api.common.ExecutionMode;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.DiscardingOutputFormat;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.optimizer.util.CompilerTestBase;
-import org.apache.flink.runtime.io.network.DataExchangeMode;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionType;
-import org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate;
-import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
-import org.apache.flink.runtime.jobgraph.JobGraph;
-import org.apache.flink.runtime.jobgraph.JobVertex;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.flink.runtime.io.network.DataExchangeMode.BATCH;
-import static org.apache.flink.runtime.io.network.DataExchangeMode.PIPELINED;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * This tests a fix for FLINK-2540.
- *
- * <p> This test is necessary, because {@link NAryUnionPlanNode}s are not directly translated
- * to runtime tasks by the {@link JobGraphGenerator}. Instead, the network stack unions the
- * inputs by directly reading from multiple inputs (via {@link UnionInputGate}).
- *
- * <pre>
- * (source)-\ /-\
- * (union)-+ (join)
- * (source)-/ \-/
- * </pre>
- *
- * @see <a href="https://issues.apache.org/jira/browse/FLINK-2540">FLINK-2540</a>
- */
-@RunWith(Parameterized.class)
-@SuppressWarnings({"serial","unchecked"})
-public class UnionClosedBranchingTest extends CompilerTestBase {
-
- @Parameterized.Parameters
- public static Collection<Object[]> params() {
- Collection<Object[]> params = Arrays.asList(new Object[][]{
- {ExecutionMode.PIPELINED, PIPELINED, BATCH},
- {ExecutionMode.PIPELINED_FORCED, PIPELINED, PIPELINED},
- {ExecutionMode.BATCH, BATCH, BATCH},
- {ExecutionMode.BATCH_FORCED, BATCH, BATCH},
- });
-
- // Make sure that changes to ExecutionMode are reflected in this test.
- assertEquals(ExecutionMode.values().length, params.size());
-
- return params;
- }
-
- private final ExecutionMode executionMode;
-
- /** Expected {@link DataExchangeMode} from sources to union. */
- private final DataExchangeMode sourceToUnion;
-
- /** Expected {@link DataExchangeMode} from union to join. */
- private final DataExchangeMode unionToJoin;
-
- public UnionClosedBranchingTest(
- ExecutionMode executionMode,
- DataExchangeMode sourceToUnion,
- DataExchangeMode unionToJoin) {
-
- this.executionMode = executionMode;
- this.sourceToUnion = sourceToUnion;
- this.unionToJoin = unionToJoin;
- }
-
- @Test
- public void testUnionClosedBranchingTest() throws Exception {
-
- // -----------------------------------------------------------------------------------------
- // Build test program
- // -----------------------------------------------------------------------------------------
-
- ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().setExecutionMode(executionMode);
- env.setParallelism(4);
-
- DataSet<Tuple1<Integer>> src1 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
-
- DataSet<Tuple1<Integer>> src2 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
-
- DataSet<Tuple1<Integer>> union = src1.union(src2);
-
- DataSet<Tuple2<Integer, Integer>> join = union
- .join(union).where(0).equalTo(0)
- .projectFirst(0).projectSecond(0);
-
- join.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
-
- // -----------------------------------------------------------------------------------------
- // Verify optimized plan
- // -----------------------------------------------------------------------------------------
-
- OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());
-
- SinkPlanNode sinkNode = optimizedPlan.getDataSinks().iterator().next();
-
- DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
-
- // Verify that the compiler correctly sets the expected data exchange modes.
- for (Channel channel : joinNode.getInputs()) {
- assertEquals("Unexpected data exchange mode between union and join node.",
- unionToJoin, channel.getDataExchangeMode());
- }
-
- for (SourcePlanNode src : optimizedPlan.getDataSources()) {
- for (Channel channel : src.getOutgoingChannels()) {
- assertEquals("Unexpected data exchange mode between source and union node.",
- sourceToUnion, channel.getDataExchangeMode());
- }
- }
-
- // -----------------------------------------------------------------------------------------
- // Verify generated JobGraph
- // -----------------------------------------------------------------------------------------
-
- JobGraphGenerator jgg = new JobGraphGenerator();
- JobGraph jobGraph = jgg.compileJobGraph(optimizedPlan);
-
- List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
-
- // Sanity check for the test setup
- assertEquals("Unexpected number of vertices created.", 4, vertices.size());
-
- // Verify all sources
- JobVertex[] sources = new JobVertex[]{vertices.get(0), vertices.get(1)};
-
- for (JobVertex src : sources) {
- // Sanity check
- assertTrue("Unexpected vertex type. Test setup is broken.", src.isInputVertex());
-
- // The union is not translated to an extra union task, but the join uses a union
- // input gate to read multiple inputs. The source create a single result per consumer.
- assertEquals("Unexpected number of created results.", 2,
- src.getNumberOfProducedIntermediateDataSets());
-
- for (IntermediateDataSet dataSet : src.getProducedDataSets()) {
- ResultPartitionType dsType = dataSet.getResultType();
-
- // The result type is determined by the channel between the union and the join node
- // and *not* the channel between source and union.
- if (unionToJoin.equals(BATCH)) {
- assertTrue("Expected batch exchange, but result type is " + dsType + ".",
- dsType.isBlocking());
- } else {
- assertFalse("Expected non-batch exchange, but result type is " + dsType + ".",
- dsType.isBlocking());
- }
- }
- }
- }
-
-}