You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gr...@apache.org on 2017/07/12 23:44:04 UTC
[04/22] flink git commit: [FLINK-6731] [tests] Activate strict
checkstyle for flink-tests
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/ConnectedComponentsCoGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/ConnectedComponentsCoGroupTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/ConnectedComponentsCoGroupTest.java
index e134c7a..e1a5d87 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/ConnectedComponentsCoGroupTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/ConnectedComponentsCoGroupTest.java
@@ -41,11 +41,12 @@ import org.apache.flink.optimizer.plan.SourcePlanNode;
import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.CompilerTestBase;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.optimizer.util.CompilerTestBase;
import org.apache.flink.util.Collector;
+
import org.junit.Assert;
import org.junit.Test;
@@ -54,93 +55,92 @@ import org.junit.Test;
*/
@SuppressWarnings("serial")
public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
-
+
private static final String VERTEX_SOURCE = "Vertices";
-
+
private static final String ITERATION_NAME = "Connected Components Iteration";
-
+
private static final String EDGES_SOURCE = "Edges";
private static final String JOIN_NEIGHBORS_MATCH = "Join Candidate Id With Neighbor";
private static final String MIN_ID_AND_UPDATE = "Min Id and Update";
-
+
private static final String SINK = "Result";
-
+
private static final boolean PRINT_PLAN = false;
-
+
private final FieldList set0 = new FieldList(0);
-
-
+
@Test
public void testWorksetConnectedComponents() {
Plan plan = getConnectedComponentsCoGroupPlan();
plan.setExecutionConfig(new ExecutionConfig());
OptimizedPlan optPlan = compileNoStats(plan);
OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);
-
+
if (PRINT_PLAN) {
PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
String json = dumper.getOptimizerPlanAsJSON(optPlan);
System.out.println(json);
}
-
+
SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
SinkPlanNode sink = or.getNode(SINK);
WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
-
+
DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE);
-
+
// --------------------------------------------------------------------
// Plan validation:
//
// We expect the plan to go with a sort-merge join, because the CoGroup
// sorts and the join in the successive iteration can re-exploit the sorting.
// --------------------------------------------------------------------
-
+
// test all drivers
Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
-
+
Assert.assertEquals(DriverStrategy.INNER_MERGE, neighborsJoin.getDriverStrategy());
Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
-
+
Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy());
Assert.assertEquals(set0, cogroup.getKeysForInput1());
Assert.assertEquals(set0, cogroup.getKeysForInput2());
-
+
// test all the shipping strategies
Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
-
+
Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges
Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached());
-
+
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy()); // min id
Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy()); // solution set
-
+
// test all the local strategies
Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
-
+
// the sort for the neighbor join in the first iteration is pushed out of the loop
Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset
Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy()); // edges
-
+
Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy());
Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy()); // solution set
-
+
// check the caches
Assert.assertTrue(TempMode.CACHED == neighborsJoin.getInput2().getTempMode());
-
+
JobGraphGenerator jgg = new JobGraphGenerator();
jgg.compileJobGraph(optPlan);
}
@@ -150,17 +150,17 @@ public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
PreviewPlanEnvironment env = new PreviewPlanEnvironment();
env.setAsContext();
try {
- ConnectedComponentsWithCoGroup(new String[]{DEFAULT_PARALLELISM_STRING, IN_FILE, IN_FILE, OUT_FILE, "100"});
+ connectedComponentsWithCoGroup(new String[]{DEFAULT_PARALLELISM_STRING, IN_FILE, IN_FILE, OUT_FILE, "100"});
} catch (ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
- Assert.fail("ConnectedComponentsWithCoGroup failed with an exception");
+ Assert.fail("connectedComponentsWithCoGroup failed with an exception");
}
return env.getPlan();
}
- public static void ConnectedComponentsWithCoGroup(String[] args) throws Exception {
+ public static void connectedComponentsWithCoGroup(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(Integer.parseInt(args[0]));
@@ -170,8 +170,8 @@ public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
DataSet<Tuple2<Long, Long>> verticesWithId = initialVertices.flatMap(new DummyMapFunction());
- DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration
- = verticesWithId.iterateDelta(verticesWithId, Integer.parseInt(args[4]), 0).name(ITERATION_NAME);
+ DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration =
+ verticesWithId.iterateDelta(verticesWithId, Integer.parseInt(args[4]), 0).name(ITERATION_NAME);
DataSet<Tuple2<Long, Long>> joinWithNeighbors = iteration.getWorkset().join(edges)
.where(0).equalTo(0)
@@ -186,14 +186,14 @@ public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
env.execute();
}
- public static class DummyMapFunction implements FlatMapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
+ private static class DummyMapFunction implements FlatMapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
@Override
public void flatMap(Tuple1<Long> value, Collector<Tuple2<Long, Long>> out) throws Exception {
// won't be executed
}
}
- public static class DummyJoinFunction implements FlatJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ private static class DummyJoinFunction implements FlatJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public void join(Tuple2<Long, Long> first, Tuple2<Long, Long> second, Collector<Tuple2<Long, Long>> out) throws Exception {
// won't be executed
@@ -202,7 +202,7 @@ public class ConnectedComponentsCoGroupTest extends CompilerTestBase {
@ForwardedFieldsFirst("f0->f0")
@ForwardedFieldsSecond("f0->f0")
- public static class DummyCoGroupFunction implements CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ private static class DummyCoGroupFunction implements CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple2<Long, Long>> second, Collector<Tuple2<Long, Long>> out) throws Exception {
// won't be executed
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/MultipleJoinsWithSolutionSetCompilerTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
index f17b28a..f4f35e7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
@@ -18,68 +18,72 @@
package org.apache.flink.test.optimizer.iterations;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.aggregation.Aggregations;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.functions.RichJoinFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.aggregation.Aggregations;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.operators.DeltaIteration;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.optimizer.util.CompilerTestBase;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration;
-import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.plan.DualInputPlanNode;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plan.SolutionSetPlanNode;
import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+/**
+ * Test multiple joins with the solution set.
+ */
@SuppressWarnings("serial")
public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
-
+
private static final String JOIN_1 = "join1";
private static final String JOIN_2 = "join2";
-
+
@Test
public void testMultiSolutionSetJoinPlan() {
try {
-
+
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+
@SuppressWarnings("unchecked")
DataSet<Tuple2<Long, Double>> inputData = env.fromElements(new Tuple2<Long, Double>(1L, 1.0));
DataSet<Tuple2<Long, Double>> result = constructPlan(inputData, 10);
-
+
// add two sinks, to test the case of branching after an iteration
result.output(new DiscardingOutputFormat<Tuple2<Long, Double>>());
result.output(new DiscardingOutputFormat<Tuple2<Long, Double>>());
-
+
Plan p = env.createProgramPlan();
-
+
OptimizedPlan optPlan = compileNoStats(p);
-
+
OptimizerPlanNodeResolver or = getOptimizerPlanNodeResolver(optPlan);
-
+
DualInputPlanNode join1 = or.getNode(JOIN_1);
DualInputPlanNode join2 = or.getNode(JOIN_2);
-
+
assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, join1.getDriverStrategy());
assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, join2.getDriverStrategy());
-
+
assertEquals(ShipStrategyType.PARTITION_HASH, join1.getInput2().getShipStrategy());
assertEquals(ShipStrategyType.PARTITION_HASH, join2.getInput1().getShipStrategy());
-
+
assertEquals(SolutionSetPlanNode.class, join1.getInput1().getSource().getClass());
assertEquals(SolutionSetPlanNode.class, join2.getInput2().getSource().getClass());
-
+
new JobGraphGenerator().compileJobGraph(optPlan);
}
catch (Exception e) {
@@ -88,42 +92,40 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
fail("Test erroneous: " + e.getMessage());
}
}
-
-
-
+
public static DataSet<Tuple2<Long, Double>> constructPlan(DataSet<Tuple2<Long, Double>> initialData, int numIterations) {
DeltaIteration<Tuple2<Long, Double>, Tuple2<Long, Double>> iteration = initialData.iterateDelta(initialData, numIterations, 0);
-
+
DataSet<Tuple2<Long, Double>> delta = iteration.getSolutionSet()
.join(iteration.getWorkset().flatMap(new Duplicator())).where(0).equalTo(0).with(new SummingJoin()).name(JOIN_1)
.groupBy(0).aggregate(Aggregations.MIN, 1).map(new Expander())
.join(iteration.getSolutionSet()).where(0).equalTo(0).with(new SummingJoinProject()).name(JOIN_2);
-
+
DataSet<Tuple2<Long, Double>> changes = delta.groupBy(0).aggregate(Aggregations.SUM, 1);
-
+
DataSet<Tuple2<Long, Double>> result = iteration.closeWith(delta, changes);
-
+
return result;
}
-
- public static final class SummingJoin extends RichJoinFunction<Tuple2<Long, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
+
+ private static final class SummingJoin extends RichJoinFunction<Tuple2<Long, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public Tuple2<Long, Double> join(Tuple2<Long, Double> first, Tuple2<Long, Double> second) {
return new Tuple2<Long, Double>(first.f0, first.f1 + second.f1);
}
}
-
- public static final class SummingJoinProject extends RichJoinFunction<Tuple3<Long, Double, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
+
+ private static final class SummingJoinProject extends RichJoinFunction<Tuple3<Long, Double, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public Tuple2<Long, Double> join(Tuple3<Long, Double, Double> first, Tuple2<Long, Double> second) {
return new Tuple2<Long, Double>(first.f0, first.f1 + first.f2 + second.f1);
}
}
-
- public static final class Duplicator extends RichFlatMapFunction<Tuple2<Long, Double>, Tuple2<Long, Double>> {
+
+ private static final class Duplicator extends RichFlatMapFunction<Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public void flatMap(Tuple2<Long, Double> value, Collector<Tuple2<Long, Double>> out) {
@@ -131,8 +133,8 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
out.collect(value);
}
}
-
- public static final class Expander extends RichMapFunction<Tuple2<Long, Double>, Tuple3<Long, Double, Double>> {
+
+ private static final class Expander extends RichMapFunction<Tuple2<Long, Double>, Tuple3<Long, Double, Double>> {
@Override
public Tuple3<Long, Double, Double> map(Tuple2<Long, Double> value) {
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/PageRankCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/PageRankCompilerTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/PageRankCompilerTest.java
index f916151..d7ce29d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/PageRankCompilerTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/iterations/PageRankCompilerTest.java
@@ -16,61 +16,64 @@
* limitations under the License.
*/
-
package org.apache.flink.test.optimizer.iterations;
-import static org.apache.flink.api.java.aggregation.Aggregations.SUM;
-import static org.junit.Assert.fail;
-
import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.operators.IterativeDataSet;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.optimizer.Optimizer;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.examples.java.graph.PageRank.BuildOutgoingEdgeList;
import org.apache.flink.examples.java.graph.PageRank.Dampener;
import org.apache.flink.examples.java.graph.PageRank.EpsilonFilter;
import org.apache.flink.examples.java.graph.PageRank.JoinVertexWithEdgesMatch;
import org.apache.flink.examples.java.graph.PageRank.RankAssigner;
+import org.apache.flink.optimizer.Optimizer;
+import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
+import org.apache.flink.optimizer.plan.BulkPartialSolutionPlanNode;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plan.SinkPlanNode;
+import org.apache.flink.optimizer.util.CompilerTestBase;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.optimizer.util.CompilerTestBase;
+
import org.junit.Assert;
import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
+import static org.apache.flink.api.java.aggregation.Aggregations.SUM;
+import static org.junit.Assert.fail;
+
+/**
+ * Test compilation of PageRank implementation.
+ */
public class PageRankCompilerTest extends CompilerTestBase{
-
+
@Test
public void testPageRank() {
try {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
+
// get input data
- DataSet<Long> pagesInput = env.fromElements(1l);
+ DataSet<Long> pagesInput = env.fromElements(1L);
@SuppressWarnings("unchecked")
- DataSet<Tuple2<Long, Long>> linksInput =env.fromElements(new Tuple2<Long, Long>(1l, 2l));
-
+ DataSet<Tuple2<Long, Long>> linksInput = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
+
// assign initial rank to pages
DataSet<Tuple2<Long, Double>> pagesWithRanks = pagesInput.
map(new RankAssigner((1.0d / 10)));
-
+
// build adjacency list from link input
- DataSet<Tuple2<Long, Long[]>> adjacencyListInput =
+ DataSet<Tuple2<Long, Long[]>> adjacencyListInput =
linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList());
-
+
// set iterative data set
IterativeDataSet<Tuple2<Long, Double>> iteration = pagesWithRanks.iterate(10);
-
+
Configuration cfg = new Configuration();
cfg.setString(Optimizer.HINT_LOCAL_STRATEGY, Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND);
-
+
DataSet<Tuple2<Long, Double>> newRanks = iteration
// join pages with outgoing edges and distribute rank
.join(adjacencyListInput).where(0).equalTo(0).withParameters(cfg)
@@ -79,26 +82,26 @@ public class PageRankCompilerTest extends CompilerTestBase{
.groupBy(0).aggregate(SUM, 1)
// apply dampening factor
.map(new Dampener(0.85, 10));
-
+
DataSet<Tuple2<Long, Double>> finalPageRanks = iteration.closeWith(
- newRanks,
+ newRanks,
newRanks.join(iteration).where(0).equalTo(0)
// termination condition
.filter(new EpsilonFilter()));
-
+
finalPageRanks.output(new DiscardingOutputFormat<Tuple2<Long, Double>>());
-
+
// get the plan and compile it
Plan p = env.createProgramPlan();
OptimizedPlan op = compileNoStats(p);
-
+
SinkPlanNode sinkPlanNode = (SinkPlanNode) op.getDataSinks().iterator().next();
BulkIterationPlanNode iterPlanNode = (BulkIterationPlanNode) sinkPlanNode.getInput().getSource();
-
+
// check that the partitioning is pushed out of the first loop
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iterPlanNode.getInput().getShipStrategy());
Assert.assertEquals(LocalStrategy.NONE, iterPlanNode.getInput().getLocalStrategy());
-
+
BulkPartialSolutionPlanNode partSolPlanNode = iterPlanNode.getPartialSolutionPlanNode();
Assert.assertEquals(ShipStrategyType.FORWARD, partSolPlanNode.getOutgoingChannels().get(0).getShipStrategy());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
index 6932fd2..d066d8e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/DumpCompiledPlanTest.java
@@ -18,30 +18,31 @@
package org.apache.flink.test.optimizer.jsonplan;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.Plan;
import org.apache.flink.client.program.OptimizerPlanEnvironment;
import org.apache.flink.client.program.PreviewPlanEnvironment;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.examples.java.clustering.KMeans;
import org.apache.flink.examples.java.graph.ConnectedComponents;
import org.apache.flink.examples.java.graph.PageRank;
import org.apache.flink.examples.java.relational.TPCHQuery3;
import org.apache.flink.examples.java.relational.WebLogAnalysis;
import org.apache.flink.examples.java.wordcount.WordCount;
+import org.apache.flink.optimizer.plan.OptimizedPlan;
+import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.optimizer.util.CompilerTestBase;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
import org.junit.Assert;
import org.junit.Test;
-/*
+/**
* The tests in this class simply invokes the JSON dump code for the optimized plan.
*/
public class DumpCompiledPlanTest extends CompilerTestBase {
-
+
@Test
public void dumpWordCount() {
// prepare the test environment
@@ -51,7 +52,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
WordCount.main(new String[] {
"--input", IN_FILE,
"--output", OUT_FILE});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -59,7 +60,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpTPCH3() {
// prepare the test environment
@@ -71,7 +72,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
"--customer", IN_FILE,
"--orders", OUT_FILE,
"--output", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -79,7 +80,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpIterativeKMeans() {
// prepare the test environment
@@ -91,7 +92,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
"--centroids ", IN_FILE,
"--output ", OUT_FILE,
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -99,7 +100,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpWebLogAnalysis() {
// prepare the test environment
@@ -111,7 +112,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
"--ranks", IN_FILE,
"--visits", OUT_FILE,
"--output", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -131,7 +132,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
"--edges", IN_FILE,
"--output", OUT_FILE,
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -139,7 +140,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpPageRank() {
// prepare the test environment
@@ -152,7 +153,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
"--output", OUT_FILE,
"--numPages", "10",
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -160,7 +161,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
private void dump(Plan p) {
p.setExecutionConfig(new ExecutionConfig());
try {
@@ -168,7 +169,7 @@ public class DumpCompiledPlanTest extends CompilerTestBase {
PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
String json = dumper.getOptimizerPlanAsJSON(op);
JsonParser parser = new JsonFactory().createJsonParser(json);
- while (parser.nextToken() != null);
+ while (parser.nextToken() != null) {}
} catch (JsonParseException e) {
e.printStackTrace();
Assert.fail("JSON Generator produced malformatted output: " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/JsonJobGraphGenerationTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/JsonJobGraphGenerationTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/JsonJobGraphGenerationTest.java
index 0c14d82..3174801 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/JsonJobGraphGenerationTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/JsonJobGraphGenerationTest.java
@@ -18,12 +18,6 @@
package org.apache.flink.test.optimizer.jsonplan;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -37,10 +31,14 @@ import org.apache.flink.examples.java.wordcount.WordCount;
import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.plan.OptimizedPlan;
import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.runtime.jobgraph.jsonplan.JsonPlanGenerator;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,8 +49,14 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+/**
+ * Test job graph generation in JSON format.
+ */
public class JsonJobGraphGenerationTest {
private PrintStream out;
@@ -67,11 +71,11 @@ public class JsonJobGraphGenerationTest {
@Override
public void write(int b) {}
};
-
+
System.setOut(new PrintStream(discards));
System.setErr(new PrintStream(discards));
}
-
+
@After
public void restoreStreams() {
if (out != null) {
@@ -81,8 +85,7 @@ public class JsonJobGraphGenerationTest {
System.setOut(err);
}
}
-
-
+
@Test
public void testWordCountPlan() {
try {
@@ -95,13 +98,13 @@ public class JsonJobGraphGenerationTest {
WordCount.main(new String[0]);
}
catch (AbortError ignored) {}
-
+
// with arguments
try {
final int parallelism = 17;
JsonValidator validator = new GenericValidator(parallelism, 3);
TestingExecutionEnvironment.setAsNext(validator, parallelism);
-
+
String tmpDir = ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH;
WordCount.main(new String[] {
"--input", tmpDir,
@@ -186,7 +189,7 @@ public class JsonJobGraphGenerationTest {
fail(e.getMessage());
}
}
-
+
@Test
public void testConnectedComponents() {
try {
@@ -214,7 +217,7 @@ public class JsonJobGraphGenerationTest {
"--iterations", "100");
}
catch (AbortError ignored) {}
-
+
}
catch (Exception e) {
restoreStreams();
@@ -224,12 +227,12 @@ public class JsonJobGraphGenerationTest {
}
// ------------------------------------------------------------------------
-
- private static interface JsonValidator {
-
+
+ private interface JsonValidator {
+
void validateJson(String json) throws Exception;
}
-
+
private static class GenericValidator implements JsonValidator {
private final int expectedParallelism;
@@ -243,7 +246,7 @@ public class JsonJobGraphGenerationTest {
@Override
public void validateJson(String json) throws Exception {
final Map<String, JsonNode> idToNode = new HashMap<>();
-
+
// validate the produced JSON
ObjectMapper m = new ObjectMapper();
JsonNode rootNode = m.readTree(json);
@@ -251,7 +254,7 @@ public class JsonJobGraphGenerationTest {
JsonNode idField = rootNode.get("jid");
JsonNode nameField = rootNode.get("name");
JsonNode arrayField = rootNode.get("nodes");
-
+
assertNotNull(idField);
assertNotNull(nameField);
assertNotNull(arrayField);
@@ -260,14 +263,15 @@ public class JsonJobGraphGenerationTest {
assertTrue(arrayField.isArray());
ArrayNode array = (ArrayNode) arrayField;
- for (Iterator<JsonNode> iter = array.elements(); iter.hasNext(); ) {
+ Iterator<JsonNode> iter = array.elements();
+ while (iter.hasNext()) {
JsonNode vertex = iter.next();
JsonNode vertexIdField = vertex.get("id");
JsonNode parallelismField = vertex.get("parallelism");
JsonNode contentsFields = vertex.get("description");
JsonNode operatorField = vertex.get("operator");
-
+
assertNotNull(vertexIdField);
assertTrue(vertexIdField.isTextual());
assertNotNull(parallelismField);
@@ -276,30 +280,31 @@ public class JsonJobGraphGenerationTest {
assertTrue(contentsFields.isTextual());
assertNotNull(operatorField);
assertTrue(operatorField.isTextual());
-
+
if (contentsFields.asText().startsWith("Sync")) {
assertEquals(1, parallelismField.asInt());
}
else {
assertEquals(expectedParallelism, parallelismField.asInt());
}
-
+
idToNode.put(vertexIdField.asText(), vertex);
}
-
+
assertEquals(numNodes, idToNode.size());
-
+
// check that all inputs are contained
for (JsonNode node : idToNode.values()) {
JsonNode inputsField = node.get("inputs");
if (inputsField != null) {
- for (Iterator<JsonNode> inputsIter = inputsField.elements(); inputsIter.hasNext(); ) {
+ Iterator<JsonNode> inputsIter = inputsField.elements();
+ while (inputsIter.hasNext()) {
JsonNode inputNode = inputsIter.next();
JsonNode inputIdField = inputNode.get("id");
-
+
assertNotNull(inputIdField);
assertTrue(inputIdField.isTextual());
-
+
String inputIdString = inputIdField.asText();
assertTrue(idToNode.containsKey(inputIdString));
}
@@ -309,15 +314,15 @@ public class JsonJobGraphGenerationTest {
}
// ------------------------------------------------------------------------
-
+
private static class AbortError extends Error {
private static final long serialVersionUID = 152179957828703919L;
}
-
+
// ------------------------------------------------------------------------
-
+
private static class TestingExecutionEnvironment extends ExecutionEnvironment {
-
+
private final JsonValidator validator;
private TestingExecutionEnvironment(JsonValidator validator) {
@@ -331,7 +336,7 @@ public class JsonJobGraphGenerationTest {
@Override
public JobExecutionResult execute(String jobName) throws Exception {
Plan plan = createProgramPlan(jobName);
-
+
Optimizer pc = new Optimizer(new Configuration());
OptimizedPlan op = pc.compile(plan);
@@ -339,13 +344,13 @@ public class JsonJobGraphGenerationTest {
JobGraph jobGraph = jgg.compileJobGraph(op);
String jsonPlan = JsonPlanGenerator.generatePlan(jobGraph);
-
+
// first check that the JSON is valid
JsonParser parser = new JsonFactory().createJsonParser(jsonPlan);
- while (parser.nextToken() != null);
-
+ while (parser.nextToken() != null) {}
+
validator.validateJson(jsonPlan);
-
+
throw new AbortError();
}
@@ -353,7 +358,7 @@ public class JsonJobGraphGenerationTest {
public String getExecutionPlan() throws Exception {
throw new UnsupportedOperationException();
}
-
+
public static void setAsNext(final JsonValidator validator, final int defaultParallelism) {
initializeContextEnvironment(new ExecutionEnvironmentFactory() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
index 14f6656..a487eb0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/optimizer/jsonplan/PreviewPlanDumpTest.java
@@ -18,11 +18,6 @@
package org.apache.flink.test.optimizer.jsonplan;
-import java.util.List;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
import org.apache.flink.api.common.Plan;
import org.apache.flink.client.program.OptimizerPlanEnvironment;
import org.apache.flink.client.program.PreviewPlanEnvironment;
@@ -35,14 +30,20 @@ import org.apache.flink.optimizer.Optimizer;
import org.apache.flink.optimizer.dag.DataSinkNode;
import org.apache.flink.optimizer.plandump.PlanJSONDumpGenerator;
import org.apache.flink.optimizer.util.CompilerTestBase;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
import org.junit.Assert;
import org.junit.Test;
-/*
+import java.util.List;
+
+/**
* The tests in this class simply invokes the JSON dump code for the original plan.
*/
public class PreviewPlanDumpTest extends CompilerTestBase {
-
+
@Test
public void dumpWordCount() {
// prepare the test environment
@@ -52,7 +53,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
WordCount.main(new String[] {
"--input", IN_FILE,
"--output", OUT_FILE});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -60,7 +61,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpTPCH3() {
// prepare the test environment
@@ -72,7 +73,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
"--customer", IN_FILE,
"--orders", OUT_FILE,
"--output", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -80,7 +81,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpIterativeKMeans() {
// prepare the test environment
@@ -92,7 +93,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
"--centroids ", IN_FILE,
"--output ", OUT_FILE,
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -100,7 +101,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpWebLogAnalysis() {
// prepare the test environment
@@ -112,7 +113,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
"--ranks", IN_FILE,
"--visits", OUT_FILE,
"--output", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -132,7 +133,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
"--edges", IN_FILE,
"--output", OUT_FILE,
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -140,7 +141,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
@Test
public void dumpPageRank() {
// prepare the test environment
@@ -154,7 +155,7 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
"--output", OUT_FILE,
"--numPages", "10",
"--iterations", "123"});
- } catch(OptimizerPlanEnvironment.ProgramAbortException pae) {
+ } catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
// all good.
} catch (Exception e) {
e.printStackTrace();
@@ -162,14 +163,14 @@ public class PreviewPlanDumpTest extends CompilerTestBase {
}
dump(env.getPlan());
}
-
+
private void dump(Plan p) {
try {
List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(p);
PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
String json = dumper.getPactPlanAsJSON(sinks);
try (JsonParser parser = new JsonFactory().createParser(json)) {
- while (parser.nextToken() != null) ;
+ while (parser.nextToken() != null) {}
}
} catch (JsonParseException e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java b/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
index f012a47..21e3b4c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/AbstractQueryableStateITCase.java
@@ -18,11 +18,6 @@
package org.apache.flink.test.query;
-import akka.actor.ActorSystem;
-import akka.dispatch.Futures;
-import akka.dispatch.OnSuccess;
-import akka.dispatch.Recover;
-import akka.pattern.Patterns;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.functions.FoldFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
@@ -60,15 +55,16 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
+
+import akka.actor.ActorSystem;
+import akka.dispatch.Futures;
+import akka.dispatch.OnSuccess;
+import akka.dispatch.Recover;
+import akka.pattern.Patterns;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.Deadline;
-import scala.concurrent.duration.FiniteDuration;
-import scala.reflect.ClassTag$;
import java.util.ArrayList;
import java.util.List;
@@ -78,6 +74,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongArray;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.Deadline;
+import scala.concurrent.duration.FiniteDuration;
+import scala.reflect.ClassTag$;
+
import static org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.JobStatusIs;
import static org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobStatus;
import static org.junit.Assert.assertEquals;
@@ -92,7 +94,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
private static final FiniteDuration TEST_TIMEOUT = new FiniteDuration(100, TimeUnit.SECONDS);
private static final FiniteDuration QUERY_RETRY_DELAY = new FiniteDuration(100, TimeUnit.MILLISECONDS);
- private static ActorSystem TEST_ACTOR_SYSTEM;
+ private static ActorSystem testActorSystem;
private static final int NUM_TMS = 2;
private static final int NUM_SLOTS_PER_TM = 4;
@@ -123,7 +125,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
cluster = new TestingCluster(config, false);
cluster.start(true);
- TEST_ACTOR_SYSTEM = AkkaUtils.createDefaultActorSystem();
+ testActorSystem = AkkaUtils.createDefaultActorSystem();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
@@ -139,8 +141,8 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
fail(e.getMessage());
}
- if (TEST_ACTOR_SYSTEM != null) {
- TEST_ACTOR_SYSTEM.shutdown();
+ if (testActorSystem != null) {
+ testActorSystem.shutdown();
}
}
@@ -163,7 +165,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
* number of keys is in fixed in range 0...numKeys). The records are keyed and
* a reducing queryable state instance is created, which sums up the records.
*
- * After submitting the job in detached mode, the QueryableStateCLient is used
+ * <p>After submitting the job in detached mode, the QueryableStateCLient is used
* to query the counts of each key in rounds until all keys have non-zero counts.
*/
@Test
@@ -255,14 +257,14 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
counts.set(key, result.f1);
assertEquals("Key mismatch", key, result.f0.intValue());
}
- }, TEST_ACTOR_SYSTEM.dispatcher());
+ }, testActorSystem.dispatcher());
futures.add(result);
}
Future<Iterable<Tuple2<Integer, Long>>> futureSequence = Futures.sequence(
futures,
- TEST_ACTOR_SYSTEM.dispatcher());
+ testActorSystem.dispatcher());
Await.ready(futureSequence, deadline.timeLeft());
}
@@ -652,7 +654,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
new ValueStateDescriptor<>(
"any",
source.getType(),
- Tuple2.of(0, 1337l));
+ Tuple2.of(0, 1337L));
// only expose key "1"
QueryableStateStream<Integer, Tuple2<Integer, Long>>
@@ -710,7 +712,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
* queried. The tests succeeds after each subtask index is queried with
* value numElements (the latest element updated the state).
*
- * This is the same as the simple value state test, but uses the API shortcut.
+ * <p>This is the same as the simple value state test, but uses the API shortcut.
*/
@Test
public void testValueStateShortcut() throws Exception {
@@ -968,8 +970,8 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
// fail this test.
return Patterns.after(
retryDelay,
- TEST_ACTOR_SYSTEM.scheduler(),
- TEST_ACTOR_SYSTEM.dispatcher(),
+ testActorSystem.scheduler(),
+ testActorSystem.dispatcher(),
new Callable<Future<V>>() {
@Override
public Future<V> call() throws Exception {
@@ -986,7 +988,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
});
}
}
- }, TEST_ACTOR_SYSTEM.dispatcher());
+ }, testActorSystem.dispatcher());
}
@@ -1015,8 +1017,8 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
// fail this test.
return Patterns.after(
retryDelay,
- TEST_ACTOR_SYSTEM.scheduler(),
- TEST_ACTOR_SYSTEM.dispatcher(),
+ testActorSystem.scheduler(),
+ testActorSystem.dispatcher(),
new Callable<Future<V>>() {
@Override
public Future<V> call() throws Exception {
@@ -1033,7 +1035,7 @@ public abstract class AbstractQueryableStateITCase extends TestLogger {
});
}
}
- }, TEST_ACTOR_SYSTEM.dispatcher());
+ }, testActorSystem.dispatcher());
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java b/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
index 0f99afb..0c628e4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/KVStateRequestSerializerRocksDBTest.java
@@ -27,18 +27,16 @@ import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.contrib.streaming.state.PredefinedOptions;
import org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackend;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
-import org.apache.flink.runtime.query.netty.message.KvStateRequestSerializer;
import org.apache.flink.runtime.query.netty.message.KvStateRequestSerializerTest;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.VoidNamespace;
import org.apache.flink.runtime.state.VoidNamespaceSerializer;
import org.apache.flink.runtime.state.internal.InternalListState;
-
import org.apache.flink.runtime.state.internal.InternalMapState;
+
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
@@ -48,7 +46,8 @@ import static org.mockito.Mockito.mock;
/**
* Additional tests for the serialization and deserialization of {@link
- * KvStateRequestSerializer} with a RocksDB state back-end.
+ * org.apache.flink.runtime.query.netty.message.KvStateRequestSerializer}
+ * with a RocksDB state back-end.
*/
public final class KVStateRequestSerializerRocksDBTest {
@@ -62,7 +61,7 @@ public final class KVStateRequestSerializerRocksDBTest {
*
* @param <K> key type
*/
- final static class RocksDBKeyedStateBackend2<K> extends RocksDBKeyedStateBackend<K> {
+ static final class RocksDBKeyedStateBackend2<K> extends RocksDBKeyedStateBackend<K> {
RocksDBKeyedStateBackend2(
final String operatorIdentifier,
@@ -127,7 +126,7 @@ public final class KVStateRequestSerializerRocksDBTest {
KvStateRequestSerializerTest.testListSerialization(key, listState);
}
-
+
/**
* Tests map serialization and deserialization match.
*
@@ -158,10 +157,10 @@ public final class KVStateRequestSerializerRocksDBTest {
longHeapKeyedStateBackend.restore(null);
longHeapKeyedStateBackend.setCurrentKey(key);
- final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>)
+ final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>)
longHeapKeyedStateBackend.getPartitionedState(
VoidNamespace.INSTANCE,
- VoidNamespaceSerializer.INSTANCE,
+ VoidNamespaceSerializer.INSTANCE,
new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));
KvStateRequestSerializerTest.testMapSerialization(key, mapState);
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseFsBackend.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseFsBackend.java b/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseFsBackend.java
index 6c72bca..b91d277 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseFsBackend.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseFsBackend.java
@@ -20,6 +20,7 @@ package org.apache.flink.test.query;
import org.apache.flink.runtime.state.AbstractStateBackend;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
+
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseRocksDBBackend.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseRocksDBBackend.java b/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseRocksDBBackend.java
index 4799da4..9547c5a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseRocksDBBackend.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/query/QueryableStateITCaseRocksDBBackend.java
@@ -20,6 +20,7 @@ package org.apache.flink.test.query;
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
import org.apache.flink.runtime.state.AbstractStateBackend;
+
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
index 6882b46..27bd11d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java
@@ -18,12 +18,6 @@
package org.apache.flink.test.recovery;
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.pattern.Patterns;
-import akka.util.Timeout;
-
-import org.apache.commons.io.FileUtils;
import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -42,17 +36,15 @@ import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.util.NetUtils;
import org.apache.flink.util.TestLogger;
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.pattern.Patterns;
+import akka.util.Timeout;
+import org.apache.commons.io.FileUtils;
import org.junit.Test;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Some;
-import scala.Tuple2;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.FiniteDuration;
-
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -61,6 +53,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
import static org.apache.flink.runtime.testutils.CommonTestUtils.getCurrentClasspath;
import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath;
import static org.junit.Assert.assertFalse;
@@ -70,7 +68,7 @@ import static org.junit.Assert.fail;
* Abstract base for tests verifying the behavior of the recovery in the
* case when a TaskManager fails (process is killed) in the middle of a job execution.
*
- * The test works with multiple task managers processes by spawning JVMs.
+ * <p>The test works with multiple task managers processes by spawning JVMs.
* Initially, it starts a JobManager in process and two TaskManagers JVMs with
* 2 task slots each.
* It submits a program with parallelism 4 and waits until all tasks are brought up.
@@ -81,8 +79,6 @@ import static org.junit.Assert.fail;
*/
public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends TestLogger {
- protected final Logger LOG = LoggerFactory.getLogger(getClass());
-
protected static final String READY_MARKER_FILE_PREFIX = "ready_";
protected static final String PROCEED_MARKER_FILE = "proceed";
protected static final String FINISH_MARKER_FILE_PREFIX = "finish_";
@@ -293,10 +289,8 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
*/
public abstract void testTaskManagerFailure(int jobManagerPort, File coordinateDir) throws Exception;
-
protected void waitUntilNumTaskManagersAreRegistered(ActorRef jobManager, int numExpected, long maxDelayMillis)
- throws Exception
- {
+ throws Exception {
final long pollInterval = 10_000_000; // 10 ms = 10,000,000 nanos
final long deadline = System.nanoTime() + maxDelayMillis * 1_000_000;
@@ -360,7 +354,6 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test
long now = System.currentTimeMillis();
final long deadline = now + timeout;
-
while (now < deadline) {
boolean allFound = true;
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
index 0b49814..eb596d4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/FastFailuresITCase.java
@@ -18,7 +18,6 @@
package org.apache.flink.test.recovery;
-
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -26,24 +25,26 @@ import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
-
import org.apache.flink.streaming.util.TestStreamEnvironment;
import org.apache.flink.util.TestLogger;
+
import org.junit.Test;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.fail;
+/**
+ * Test program with very fast failure rate.
+ */
@SuppressWarnings("serial")
public class FastFailuresITCase extends TestLogger {
static final AtomicInteger FAILURES_SO_FAR = new AtomicInteger();
static final int NUM_FAILURES = 200;
-
+
@Test
public void testThis() {
final int parallelism = 4;
@@ -51,17 +52,17 @@ public class FastFailuresITCase extends TestLogger {
Configuration config = new Configuration();
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
-
+
LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(config, false);
cluster.start();
TestStreamEnvironment env = new TestStreamEnvironment(cluster, parallelism);
-
+
env.getConfig().disableSysoutLogging();
env.setParallelism(parallelism);
env.enableCheckpointing(1000);
env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(210, 0));
-
+
DataStream<Tuple2<Integer, Integer>> input = env.addSource(new RichSourceFunction<Tuple2<Integer, Integer>>() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
index f70cc1e..51868af 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHACheckpointRecoveryITCase.java
@@ -18,11 +18,6 @@
package org.apache.flink.test.recovery;
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.actor.PoisonPill;
-import org.apache.commons.io.FileUtils;
-import org.apache.curator.test.TestingServer;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
@@ -70,6 +65,12 @@ import org.apache.flink.testutils.junit.RetryOnFailure;
import org.apache.flink.testutils.junit.RetryRule;
import org.apache.flink.util.Collector;
import org.apache.flink.util.TestLogger;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.PoisonPill;
+import org.apache.commons.io.FileUtils;
+import org.apache.curator.test.TestingServer;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
@@ -77,11 +78,6 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Option;
-import scala.Some;
-import scala.Tuple2;
-import scala.concurrent.duration.Deadline;
-import scala.concurrent.duration.FiniteDuration;
import java.io.File;
import java.io.IOException;
@@ -93,20 +89,29 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReference;
+import scala.Option;
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.duration.Deadline;
+import scala.concurrent.duration.FiniteDuration;
+
import static org.apache.flink.runtime.messages.JobManagerMessages.SubmitJob;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
+/**
+ * Test JobManager recovery.
+ */
public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
@Rule
public RetryRule retryRule = new RetryRule();
- private final static ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
+ private static final ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
- private final static FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
+ private static final FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
private static final File FileStateBackendBasePath;
@@ -147,16 +152,16 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
private static final int Parallelism = 8;
- private static CountDownLatch CompletedCheckpointsLatch = new CountDownLatch(4);
- private static CountDownLatch CompletedCheckpointsLatch2 = new CountDownLatch(6);
+ private static CountDownLatch completedCheckpointsLatch = new CountDownLatch(4);
+ private static CountDownLatch completedCheckpointsLatch2 = new CountDownLatch(6);
- private static AtomicLongArray RecoveredStates = new AtomicLongArray(Parallelism);
+ private static AtomicLongArray recoveredStates = new AtomicLongArray(Parallelism);
- private static CountDownLatch FinalCountLatch = new CountDownLatch(1);
+ private static CountDownLatch finalCountLatch = new CountDownLatch(1);
- private static AtomicReference<Long> FinalCount = new AtomicReference<>();
+ private static AtomicReference<Long> finalCount = new AtomicReference<>();
- private static long LastElement = -1;
+ private static long lastElement = -1;
private static final int retainedCheckpoints = 2;
@@ -166,7 +171,7 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
* @see <a href="https://issues.apache.org/jira/browse/FLINK-3185">FLINK-3185</a>
*/
@Test
- @RetryOnFailure(times=1)
+ @RetryOnFailure(times = 1)
public void testCheckpointRecoveryFailure() throws Exception {
final Deadline testDeadline = TestTimeOut.fromNow();
final String zooKeeperQuorum = ZooKeeper.getConnectString();
@@ -361,7 +366,6 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, numSlots);
config.setString(CoreOptions.CHECKPOINTS_DIRECTORY, temporaryFolder.newFolder().toString());
-
String tmpFolderString = temporaryFolder.newFolder().toString();
config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, tmpFolderString);
config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
@@ -399,17 +403,17 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
JobGraph jobGraph = env.getStreamGraph().getJobGraph();
miniCluster.submitJobDetached(jobGraph);
- CompletedCheckpointsLatch.await();
+ completedCheckpointsLatch.await();
jmGateway.tell(PoisonPill.getInstance());
// Wait to finish
- FinalCountLatch.await();
+ finalCountLatch.await();
- assertEquals(expectedSum, (long) FinalCount.get());
+ assertEquals(expectedSum, (long) finalCount.get());
for (int i = 0; i < Parallelism; i++) {
- assertNotEquals(0, RecoveredStates.get(i));
+ assertNotEquals(0, recoveredStates.get(i));
}
} finally {
@@ -467,7 +471,7 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
synchronized (ctx.getCheckpointLock()) {
if (current <= end) {
ctx.collect(current++);
- } else if(repeat > 0) {
+ } else if (repeat > 0) {
--repeat;
current = 0;
} else {
@@ -481,9 +485,9 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
}
}
- CompletedCheckpointsLatch2.await();
+ completedCheckpointsLatch2.await();
synchronized (ctx.getCheckpointLock()) {
- ctx.collect(LastElement);
+ ctx.collect(lastElement);
}
}
@@ -503,7 +507,7 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
// This is necessary to make sure that something is recovered at all. Otherwise it
// might happen that the job is restarted from the beginning.
- RecoveredStates.set(getRuntimeContext().getIndexOfThisSubtask(), 1);
+ recoveredStates.set(getRuntimeContext().getIndexOfThisSubtask(), 1);
sync.countDown();
@@ -535,12 +539,12 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
@Override
public void invoke(Long value) throws Exception {
- if (value == LastElement) {
+ if (value == lastElement) {
numberOfReceivedLastElements++;
if (numberOfReceivedLastElements == Parallelism) {
- FinalCount.set(current);
- FinalCountLatch.countDown();
+ finalCount.set(current);
+ finalCountLatch.countDown();
}
else if (numberOfReceivedLastElements > Parallelism) {
throw new IllegalStateException("Received more elements than parallelism.");
@@ -572,12 +576,12 @@ public class JobManagerHACheckpointRecoveryITCase extends TestLogger {
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
LOG.debug("Checkpoint {} completed.", checkpointId);
- CompletedCheckpointsLatch.countDown();
- CompletedCheckpointsLatch2.countDown();
+ completedCheckpointsLatch.countDown();
+ completedCheckpointsLatch2.countDown();
}
}
- public static class StatefulFlatMap extends RichFlatMapFunction<Long, Long> implements CheckpointedFunction {
+ private static class StatefulFlatMap extends RichFlatMapFunction<Long, Long> implements CheckpointedFunction {
private static final long serialVersionUID = 9031079547885320663L;
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
index 3f2eea3..44b27cd 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAJobGraphRecoveryITCase.java
@@ -18,13 +18,6 @@
package org.apache.flink.test.recovery;
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.actor.Props;
-import akka.actor.UntypedActor;
-import akka.testkit.TestActorRef;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.ConfigConstants;
@@ -55,19 +48,20 @@ import org.apache.flink.runtime.testutils.JobManagerProcess;
import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
import org.apache.flink.util.TestLogger;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.Props;
+import akka.actor.UntypedActor;
+import akka.testkit.TestActorRef;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import scala.Option;
-import scala.Some;
-import scala.Tuple2;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.Deadline;
-import scala.concurrent.duration.FiniteDuration;
import java.io.File;
import java.util.Collection;
@@ -77,6 +71,14 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import scala.Option;
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.Deadline;
+import scala.concurrent.duration.FiniteDuration;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -85,9 +87,9 @@ import static org.junit.Assert.fail;
*/
public class JobManagerHAJobGraphRecoveryITCase extends TestLogger {
- private final static ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
+ private static final ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
- private final static FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
+ private static final FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
index 2983d66..2820dd2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureBatchRecoveryITCase.java
@@ -18,9 +18,6 @@
package org.apache.flink.test.recovery;
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.ExecutionMode;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
@@ -49,14 +46,15 @@ import org.apache.flink.runtime.testutils.ZooKeeperTestUtils;
import org.apache.flink.runtime.zookeeper.ZooKeeperTestEnvironment;
import org.apache.flink.util.Collector;
import org.apache.flink.util.TestLogger;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import scala.Option;
-import scala.concurrent.duration.Deadline;
-import scala.concurrent.duration.FiniteDuration;
import java.io.File;
import java.io.IOException;
@@ -65,6 +63,10 @@ import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import scala.Option;
+import scala.concurrent.duration.Deadline;
+import scala.concurrent.duration.FiniteDuration;
+
import static org.apache.flink.runtime.testutils.CommonTestUtils.createTempDirectory;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -88,9 +90,9 @@ import static org.junit.Assert.fail;
@RunWith(Parameterized.class)
public class JobManagerHAProcessFailureBatchRecoveryITCase extends TestLogger {
- private final static ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
+ private static final ZooKeeperTestEnvironment ZooKeeper = new ZooKeeperTestEnvironment(1);
- private final static FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
+ private static final FiniteDuration TestTimeOut = new FiniteDuration(5, TimeUnit.MINUTES);
private static final File FileStateBackendBasePath;
@@ -164,8 +166,8 @@ public class JobManagerHAProcessFailureBatchRecoveryITCase extends TestLogger {
env.getConfig().setExecutionMode(executionMode);
env.getConfig().disableSysoutLogging();
- final long NUM_ELEMENTS = 100000L;
- final DataSet<Long> result = env.generateSequence(1, NUM_ELEMENTS)
+ final long numElements = 100000L;
+ final DataSet<Long> result = env.generateSequence(1, numElements)
// make sure every mapper is involved (no one is skipped because of lazy split assignment)
.rebalance()
// the majority of the behavior is in the MapFunction
@@ -209,7 +211,7 @@ public class JobManagerHAProcessFailureBatchRecoveryITCase extends TestLogger {
.flatMap(new RichFlatMapFunction<Long, Long>() {
@Override
public void flatMap(Long value, Collector<Long> out) throws Exception {
- assertEquals(NUM_ELEMENTS * (NUM_ELEMENTS + 1L) / 2L, (long) value);
+ assertEquals(numElements * (numElements + 1L) / 2L, (long) value);
int taskIndex = getRuntimeContext().getIndexOfThisSubtask();
AbstractTaskManagerProcessFailureRecoveryTest.touchFile(
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
index 59d5a51..b42204d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
@@ -18,11 +18,6 @@
package org.apache.flink.test.recovery;
-import akka.actor.ActorRef;
-import akka.actor.ActorSystem;
-import akka.pattern.Patterns;
-import akka.util.Timeout;
-
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
@@ -42,15 +37,13 @@ import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.testingUtils.TestingUtils;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.util.NetUtils;
-
import org.apache.flink.util.TestLogger;
-import org.junit.Test;
-import scala.Some;
-import scala.Tuple2;
-import scala.concurrent.Await;
-import scala.concurrent.Future;
-import scala.concurrent.duration.FiniteDuration;
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.pattern.Patterns;
+import akka.util.Timeout;
+import org.junit.Test;
import java.io.File;
import java.io.StringWriter;
@@ -58,6 +51,12 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
import static org.apache.flink.runtime.testutils.CommonTestUtils.getCurrentClasspath;
import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath;
import static org.junit.Assert.assertFalse;
@@ -71,7 +70,7 @@ import static org.junit.Assert.fail;
*/
@SuppressWarnings("serial")
public class ProcessFailureCancelingITCase extends TestLogger {
-
+
@Test
public void testCancelingOnProcessFailure() throws Exception {
final StringWriter processOutput = new StringWriter();
@@ -137,14 +136,14 @@ public class ProcessFailureCancelingITCase extends TestLogger {
// start the first two TaskManager processes
taskManagerProcess = new ProcessBuilder(command).start();
new CommonTestUtils.PipeForwarder(taskManagerProcess.getErrorStream(), processOutput);
-
+
// we wait for the JobManager to have the two TaskManagers available
// since some of the CI environments are very hostile, we need to give this a lot of time (2 minutes)
waitUntilNumTaskManagersAreRegistered(jmActor, 1, 120000);
-
+
final Throwable[] errorRef = new Throwable[1];
- // start the test program, which infinitely blocks
+ // start the test program, which infinitely blocks
Runnable programRunner = new Runnable() {
@Override
public void run() {
@@ -176,7 +175,7 @@ public class ProcessFailureCancelingITCase extends TestLogger {
}
}
};
-
+
Thread programThread = new Thread(programRunner);
// kill the TaskManager
@@ -186,21 +185,21 @@ public class ProcessFailureCancelingITCase extends TestLogger {
// immediately submit the job. this should hit the case
// where the JobManager still thinks it has the TaskManager and tries to send it tasks
programThread.start();
-
+
// try to cancel the job
cancelRunningJob(jmActor);
// we should see a failure within reasonable time (10s is the ask timeout).
- // since the CI environment is often slow, we conservatively give it up to 2 minutes,
+ // since the CI environment is often slow, we conservatively give it up to 2 minutes,
// to fail, which is much lower than the failure time given by the heartbeats ( > 2000s)
-
+
programThread.join(120000);
-
+
assertFalse("The program did not cancel in time (2 minutes)", programThread.isAlive());
-
+
Throwable error = errorRef[0];
assertNotNull("The program did not fail properly", error);
-
+
assertTrue(error instanceof ProgramInvocationException);
// all seems well :-)
}
@@ -225,15 +224,15 @@ public class ProcessFailureCancelingITCase extends TestLogger {
}
}
}
-
+
private void cancelRunningJob(ActorRef jobManager) throws Exception {
final FiniteDuration askTimeout = new FiniteDuration(10, TimeUnit.SECONDS);
-
+
// try at most for 30 seconds
final long deadline = System.currentTimeMillis() + 30000;
JobID jobId = null;
-
+
do {
Future<Object> response = Patterns.ask(jobManager,
JobManagerMessages.getRequestRunningJobsStatus(), new Timeout(askTimeout));
@@ -247,9 +246,9 @@ public class ProcessFailureCancelingITCase extends TestLogger {
}
if (result instanceof JobManagerMessages.RunningJobsStatus) {
-
+
List<JobStatusMessage> jobs = ((JobManagerMessages.RunningJobsStatus) result).getStatusMessages();
-
+
if (jobs.size() == 1) {
jobId = jobs.get(0).getJobId();
break;
@@ -262,7 +261,7 @@ public class ProcessFailureCancelingITCase extends TestLogger {
// we never found it running, must have failed already
return;
}
-
+
// tell the JobManager to cancel the job
jobManager.tell(
new JobManagerMessages.LeaderSessionMessage(
@@ -272,8 +271,7 @@ public class ProcessFailureCancelingITCase extends TestLogger {
}
private void waitUntilNumTaskManagersAreRegistered(ActorRef jobManager, int numExpected, long maxDelay)
- throws Exception
- {
+ throws Exception {
final long deadline = System.currentTimeMillis() + maxDelay;
while (true) {
long remaining = deadline - System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java
index a0d6b58..6fc876f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFailureRateStrategyITBase.java
@@ -21,8 +21,12 @@ package org.apache.flink.test.recovery;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+
import org.junit.BeforeClass;
+/**
+ * Test cluster configuration with failure-rate recovery.
+ */
public class SimpleRecoveryFailureRateStrategyITBase extends SimpleRecoveryITCaseBase {
@BeforeClass
public static void setupCluster() {
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFixedDelayRestartStrategyITBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFixedDelayRestartStrategyITBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFixedDelayRestartStrategyITBase.java
index f09efc5..405a1c7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFixedDelayRestartStrategyITBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryFixedDelayRestartStrategyITBase.java
@@ -21,8 +21,12 @@ package org.apache.flink.test.recovery;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+
import org.junit.BeforeClass;
+/**
+ * Test cluster configuration with fixed-delay recovery.
+ */
public class SimpleRecoveryFixedDelayRestartStrategyITBase extends SimpleRecoveryITCaseBase {
@BeforeClass
public static void setupCluster() {
http://git-wip-us.apache.org/repos/asf/flink/blob/9bd491e0/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java
index bf7c524..ceef78b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCaseBase.java
@@ -27,13 +27,17 @@ import org.apache.flink.api.java.io.LocalCollectionOutputFormat;
import org.apache.flink.client.program.ProgramInvocationException;
import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+
import org.junit.AfterClass;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* A series of tests (reusing one FlinkMiniCluster) where tasks fail (one or more time)
@@ -130,14 +134,14 @@ public abstract class SimpleRecoveryITCaseBase {
private void executeAndRunAssertions(ExecutionEnvironment env) throws Exception {
try {
- JobExecutionResult result = env.execute();
- assertTrue(result.getNetRuntime() >= 0);
- assertNotNull(result.getAllAccumulatorResults());
- assertTrue(result.getAllAccumulatorResults().isEmpty());
- }
- catch (JobExecutionException e) {
- fail("The program should have succeeded on the second run");
- }
+ JobExecutionResult result = env.execute();
+ assertTrue(result.getNetRuntime() >= 0);
+ assertNotNull(result.getAllAccumulatorResults());
+ assertTrue(result.getAllAccumulatorResults().isEmpty());
+ }
+ catch (JobExecutionException e) {
+ fail("The program should have succeeded on the second run");
+ }
}
@Test