You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2014/08/08 15:30:12 UTC

[1/3] git commit: [FLINK-909] Adjust aggregators test case

Repository: incubator-flink
Updated Branches:
  refs/heads/release-0.6 4046e6a26 -> cb81fac88


[FLINK-909] Adjust aggregators test case


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/18f13eaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/18f13eaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/18f13eaf

Branch: refs/heads/release-0.6
Commit: 18f13eaf5fb8208e6c1867b18cd5643bf86809fe
Parents: 38cbf0b
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Aug 8 14:47:43 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Fri Aug 8 15:27:30 2014 +0200

----------------------------------------------------------------------
 .../runtime/iterative/concurrent/SuperstepKickoffLatch.java     | 2 +-
 .../iterative/concurrent/SuperstepKickoffLatchBroker.java       | 1 -
 .../ConnectedComponentsWithParametrizableAggregatorITCase.java  | 5 ++++-
 3 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/18f13eaf/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
index b53928c..91b1db0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
@@ -29,7 +29,7 @@ public class SuperstepKickoffLatch {
 	public void triggerNextSuperstep() {
 		synchronized (monitor) {
 			if (terminated) {
-				throw new IllegalStateException("Already teriminated.");
+				throw new IllegalStateException("Already terminated.");
 			}
 			superstepNumber++;
 			monitor.notifyAll();

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/18f13eaf/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
index 41f6985..4439acf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 
-
 package org.apache.flink.runtime.iterative.concurrent;
 
 public class SuperstepKickoffLatchBroker extends Broker<SuperstepKickoffLatch> {

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/18f13eaf/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
index 9382708..e1ae706 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
@@ -96,11 +96,14 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
 	protected void postSubmit() throws Exception {
 		compareResultsByLinesInMemory(expectedResult, resultPath);
 		long[] aggr_values = ConnectedComponentsWithAggregatorProgram.aggr_value;
+		
+		// note that position 0 has the end result from superstep 1, retrieved at the start of iteration 2
+		// position one as superstep 2, retrieved at the start of iteration 3.
+		// the result from iteration 5 is not available, because no iteration 6 happens
 		Assert.assertEquals(3, aggr_values[0]);
 		Assert.assertEquals(4, aggr_values[1]);
 		Assert.assertEquals(5, aggr_values[2]);
 		Assert.assertEquals(6, aggr_values[3]);
-		Assert.assertEquals(6, aggr_values[4]);
 	}
 
 


[2/3] git commit: [FLINK-909] Remove additional empty (and non empty for iterative broadcast variables) superstep. [FLINK-945] Fix early memory release in iterations

Posted by rm...@apache.org.
[FLINK-909]  Remove additional empty (and non empty for iterative broadcast variables) superstep.
[FLINK-945]  Fix early memory release in iterations


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/38cbf0b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/38cbf0b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/38cbf0b9

Branch: refs/heads/release-0.6
Commit: 38cbf0b915dea175f287d7c2a627467a02d28474
Parents: 4046e6a
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Aug 7 22:00:51 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Fri Aug 8 15:27:30 2014 +0200

----------------------------------------------------------------------
 .../concurrent/SuperstepKickoffLatch.java       |  65 +++++
 .../concurrent/SuperstepKickoffLatchBroker.java |  32 +++
 .../task/AbstractIterativePactTask.java         |  41 +--
 .../iterative/task/IterationHeadPactTask.java   |  21 +-
 .../task/IterationIntermediatePactTask.java     |  31 ++-
 .../iterative/task/IterationTailPactTask.java   |  38 +--
 .../concurrent/SuperstepKickoffLatchTest.java   | 247 +++++++++++++++++++
 7 files changed, 404 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
new file mode 100644
index 0000000..b53928c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+public class SuperstepKickoffLatch {
+	
+	private final Object monitor = new Object();
+	
+	private int superstepNumber = 1;
+	
+	private boolean terminated;
+	
+	public void triggerNextSuperstep() {
+		synchronized (monitor) {
+			if (terminated) {
+				throw new IllegalStateException("Already teriminated.");
+			}
+			superstepNumber++;
+			monitor.notifyAll();
+		}
+	}
+	
+	public void signalTermination() {
+		synchronized (monitor) {
+			terminated = true;
+			monitor.notifyAll();
+		}
+	}
+	
+	public boolean awaitStartOfSuperstepOrTermination(int superstep) throws InterruptedException {
+		while (true) {
+			synchronized (monitor) {
+				if (terminated) {
+					return true;
+				}
+				else if (superstepNumber == superstep) {
+					// reached the superstep. all good!
+					return false;
+				}
+				else if (superstepNumber == superstep - 1) {
+					monitor.wait(2000);
+				}
+				else {
+					throw new IllegalStateException("Error while waiting for start of next superstep. current= " + superstepNumber + " waitingFor=" + superstep);
+				}
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
new file mode 100644
index 0000000..41f6985
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+public class SuperstepKickoffLatchBroker extends Broker<SuperstepKickoffLatch> {
+
+	private static final SuperstepKickoffLatchBroker INSTANCE = new SuperstepKickoffLatchBroker();
+
+	private SuperstepKickoffLatchBroker() {}
+
+
+	public static Broker<SuperstepKickoffLatch> instance() {
+		return INSTANCE;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 636c492..4c03278 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -210,28 +210,17 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 		return this.iterationAggregators;
 	}
 
-	protected void checkForTerminationAndResetEndOfSuperstepState() throws IOException {
+	protected void verifyEndOfSuperstepState() throws IOException {
 		// sanity check that there is at least one iterative input reader
 		if (this.iterativeInputs.length == 0 && this.iterativeBroadcastInputs.length == 0) {
-			throw new IllegalStateException();
+			throw new IllegalStateException("Error: Iterative task without a single iterative input.");
 		}
 
-		// check whether this step ended due to end-of-superstep, or proper close
-		boolean anyClosed = false;
-		boolean allClosed = true;
-
 		for (int inputNum : this.iterativeInputs) {
 			MutableReader<?> reader = this.inputReaders[inputNum];
 
-			if (reader.isInputClosed()) {
-				anyClosed = true;
-			}
-			else {
-				// check if reader has reached the end of superstep, or if the operation skipped out early
+			if (!reader.isInputClosed()) {
 				if (reader.hasReachedEndOfSuperstep()) {
-					allClosed = false;
-					
-					// also reset the end-of-superstep state
 					reader.startNextSuperstep();
 				}
 				else {
@@ -241,11 +230,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 					Object o = this.inputSerializers[inputNum].getSerializer().createInstance();
 					while ((o = inIter.next(o)) != null);
 					
-					if (reader.isInputClosed()) {
-						anyClosed = true;
-					} else {
-						allClosed = false;
-						
+					if (!reader.isInputClosed()) {
 						// also reset the end-of-superstep state
 						reader.startNextSuperstep();
 					}
@@ -256,28 +241,16 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
 		for (int inputNum : this.iterativeBroadcastInputs) {
 			MutableReader<?> reader = this.broadcastInputReaders[inputNum];
 
-			if (reader.isInputClosed()) {
-				anyClosed = true;
-			}
-			else {
-				// sanity check that the BC input is at the end of teh superstep
+			if (!reader.isInputClosed()) {
+				
+				// sanity check that the BC input is at the end of the superstep
 				if (!reader.hasReachedEndOfSuperstep()) {
 					throw new IllegalStateException("An iterative broadcast input has not been fully consumed.");
 				}
 				
-				allClosed = false;
 				reader.startNextSuperstep();
 			}
 		}
-
-		// sanity check whether we saw the same state (end-of-superstep or termination) on all inputs
-		if (allClosed != anyClosed) {
-			throw new IllegalStateException("Inconsistent state: Iteration termination received on some, but not all inputs.");
-		}
-
-		if (allClosed) {
-			requestTermination();
-		}
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index 797bbb6..3dbd47c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -41,6 +41,8 @@ import org.apache.flink.runtime.iterative.concurrent.SolutionSetBroker;
 import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
 import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
 import org.apache.flink.runtime.iterative.concurrent.SuperstepBarrier;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
 import org.apache.flink.runtime.iterative.event.AllWorkersDoneEvent;
 import org.apache.flink.runtime.iterative.event.TerminationEvent;
 import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
@@ -134,8 +136,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 	private BlockingBackChannel initBackChannel() throws Exception {
 
 		/* get the size of the memory available to the backchannel */
-		int backChannelMemoryPages = getMemoryManager().computeNumberOfPages(this.config.getRelativeBackChannelMemory
-				());
+		int backChannelMemoryPages = getMemoryManager().computeNumberOfPages(this.config.getRelativeBackChannelMemory());
 
 		/* allocate the memory available to the backchannel */
 		List<MemorySegment> segments = new ArrayList<MemorySegment>();
@@ -220,6 +221,9 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 
 		try {
 			/* used for receiving the current iteration result from iteration tail */
+			SuperstepKickoffLatch nextStepKickoff = new SuperstepKickoffLatch();
+			SuperstepKickoffLatchBroker.instance().handIn(brokerKey, nextStepKickoff);
+			
 			BlockingBackChannel backChannel = initBackChannel();
 			SuperstepBarrier barrier = initSuperstepBarrier();
 			SolutionSetUpdateBarrier solutionSetUpdateBarrier = null;
@@ -316,12 +320,15 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 							+ "]"));
 					}
 					requestTermination();
+					nextStepKickoff.signalTermination();
 				} else {
 					incrementIterationCounter();
 
 					String[] globalAggregateNames = barrier.getAggregatorNames();
 					Value[] globalAggregates = barrier.getAggregates();
 					aggregatorRegistry.updateGlobalAggregatesAndReset(globalAggregateNames, globalAggregates);
+					
+					nextStepKickoff.triggerNextSuperstep();
 				}
 			}
 
@@ -344,12 +351,10 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
 			// - solution set index
 			IterationAggregatorBroker.instance().remove(brokerKey);
 			BlockingBackChannelBroker.instance().remove(brokerKey);
-			if (isWorksetIteration) {
-				SolutionSetBroker.instance().remove(brokerKey);
-				if (waitForSolutionSetUpdate) {
-					SolutionSetUpdateBarrierBroker.instance().remove(brokerKey);
-				}
-			}
+			SuperstepKickoffLatchBroker.instance().remove(brokerKey);
+			SolutionSetBroker.instance().remove(brokerKey);
+			SolutionSetUpdateBarrierBroker.instance().remove(brokerKey);
+
 			if (solutionSet != null) {
 				solutionSet.close();
 				solutionSet = null;

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index c23eae1..b12e70b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.runtime.io.network.api.BufferWriter;
 import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
 import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
 import org.apache.flink.runtime.iterative.event.TerminationEvent;
 import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
 import org.apache.flink.util.Collector;
@@ -78,6 +80,8 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
 
 	@Override
 	public void run() throws Exception {
+		
+		SuperstepKickoffLatch nextSuperstepLatch = SuperstepKickoffLatchBroker.instance().get(brokerKey());
 
 		while (this.running && !terminationRequested()) {
 
@@ -88,26 +92,31 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
 			super.run();
 
 			// check if termination was requested
-			checkForTerminationAndResetEndOfSuperstepState();
+			verifyEndOfSuperstepState();
 
 			if (isWorksetUpdate && isWorksetIteration) {
 				long numCollected = worksetUpdateOutputCollector.getElementsCollectedAndReset();
 				worksetAggregator.aggregate(numCollected);
 			}
-
+			
 			if (log.isInfoEnabled()) {
 				log.info(formatLogString("finishing iteration [" + currentIteration() + "]"));
 			}
+			
+			// let the successors know that the end of this superstep data is reached
+			sendEndOfSuperstep();
+			
+			if (isWorksetUpdate) {
+				// notify iteration head if responsible for workset update
+				worksetBackChannel.notifyOfEndOfSuperstep();
+			}
+			
+			boolean terminated = nextSuperstepLatch.awaitStartOfSuperstepOrTermination(currentIteration() + 1);
 
-			if (!terminationRequested()) {
-				if (isWorksetUpdate) {
-					// notify iteration head if responsible for workset update
-					worksetBackChannel.notifyOfEndOfSuperstep();
-				}
-
-				// send the end-of-superstep
-				sendEndOfSuperstep();
-
+			if (terminated) {
+				requestTermination();
+			}
+			else {
 				incrementIterationCounter();
 			}
 		}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index 90d732c..0d9c903 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -23,6 +23,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
 import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
 import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
 import org.apache.flink.runtime.operators.PactTaskContext;
 import org.apache.flink.util.Collector;
@@ -95,23 +97,19 @@ public class IterationTailPactTask<S extends Function, OT> extends AbstractItera
 
 	@Override
 	public void run() throws Exception {
+		
+		SuperstepKickoffLatch nextSuperStepLatch = SuperstepKickoffLatchBroker.instance().get(brokerKey());
+		
 		while (this.running && !terminationRequested()) {
 
 			if (log.isInfoEnabled()) {
 				log.info(formatLogString("starting iteration [" + currentIteration() + "]"));
 			}
 
-			try {
-				super.run();
-			}
-			catch (NullPointerException e) {
-				boolean terminationRequested = terminationRequested();
-				System.out.println("Nullpoint exception when termination requested was " + terminationRequested);
-				e.printStackTrace();
-			}
+			super.run();
 
 			// check if termination was requested
-			checkForTerminationAndResetEndOfSuperstepState();
+			verifyEndOfSuperstepState();
 
 			if (isWorksetUpdate && isWorksetIteration) {
 				// aggregate workset update element count
@@ -123,16 +121,20 @@ public class IterationTailPactTask<S extends Function, OT> extends AbstractItera
 			if (log.isInfoEnabled()) {
 				log.info(formatLogString("finishing iteration [" + currentIteration() + "]"));
 			}
+			
+			if (isWorksetUpdate) {
+				// notify iteration head if responsible for workset update
+				worksetBackChannel.notifyOfEndOfSuperstep();
+			} else if (isSolutionSetUpdate) {
+				// notify iteration head if responsible for solution set update
+				solutionSetUpdateBarrier.notifySolutionSetUpdate();
+			}
 
-			if (!terminationRequested()) {
-				if (isWorksetUpdate) {
-					// notify iteration head if responsible for workset update
-					worksetBackChannel.notifyOfEndOfSuperstep();
-				} else if (isSolutionSetUpdate) {
-					// notify iteration head if responsible for solution set update
-					solutionSetUpdateBarrier.notifySolutionSetUpdate();
-				}
-
+			boolean terminate = nextSuperStepLatch.awaitStartOfSuperstepOrTermination(currentIteration() + 1);
+			if (terminate) {
+				requestTermination();
+			}
+			else {
 				incrementIterationCounter();
 			}
 		}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/38cbf0b9/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
new file mode 100644
index 0000000..3173570
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SuperstepKickoffLatchTest {
+
+	@Test
+	public void testWaitFromOne() {
+		try {
+			SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+			
+			Waiter w = new Waiter(latch, 2);
+			Thread waiter = new Thread(w);
+			waiter.setDaemon(true);
+			waiter.start();
+			
+			WatchDog wd = new WatchDog(waiter, 2000);
+			wd.start();
+			
+			Thread.sleep(100);
+			
+			latch.triggerNextSuperstep();
+			
+			wd.join();
+			if (wd.getError() != null) {
+				throw wd.getError();
+			}
+			
+			if (w.getError() != null) {
+				throw w.getError();
+			}
+		}
+		catch (Throwable t) {
+			t.printStackTrace();
+			Assert.fail("Error: " + t.getMessage());
+		}
+	}
+	
+	@Test
+	public void testWaitAlreadyFulfilled() {
+		try {
+			SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+			latch.triggerNextSuperstep();
+			
+			Waiter w = new Waiter(latch, 2);
+			Thread waiter = new Thread(w);
+			waiter.setDaemon(true);
+			waiter.start();
+			
+			WatchDog wd = new WatchDog(waiter, 2000);
+			wd.start();
+			
+			Thread.sleep(100);
+			
+			wd.join();
+			if (wd.getError() != null) {
+				throw wd.getError();
+			}
+			
+			if (w.getError() != null) {
+				throw w.getError();
+			}
+		}
+		catch (Throwable t) {
+			t.printStackTrace();
+			Assert.fail("Error: " + t.getMessage());
+		}
+	}
+	
+	@Test
+	public void testWaitIncorrect() {
+		try {
+			SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+			latch.triggerNextSuperstep();
+			latch.triggerNextSuperstep();
+			
+			try {
+				latch.awaitStartOfSuperstepOrTermination(2);
+				Assert.fail("should throw exception");
+			}
+			catch (IllegalStateException e) {
+				// good
+			}
+		}
+		catch (Exception e) {
+			e.printStackTrace();
+			Assert.fail("Error: " + e.getMessage());
+		}
+	}
+	
+	@Test
+	public void testWaitIncorrectAsync() {
+		try {
+			SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+			latch.triggerNextSuperstep();
+			latch.triggerNextSuperstep();
+			
+			Waiter w = new Waiter(latch, 2);
+			Thread waiter = new Thread(w);
+			waiter.setDaemon(true);
+			waiter.start();
+			
+			WatchDog wd = new WatchDog(waiter, 2000);
+			wd.start();
+			
+			Thread.sleep(100);
+			
+			wd.join();
+			if (wd.getError() != null) {
+				throw wd.getError();
+			}
+			
+			if (w.getError() != null) {
+				if (!(w.getError() instanceof IllegalStateException)) {
+					throw new Exception("wrong exception type " + w.getError());
+				}
+			} else {
+				Assert.fail("should cause exception");
+			}
+		}
+		catch (Throwable t) {
+			t.printStackTrace();
+			Assert.fail("Error: " + t.getMessage());
+		}
+	}
+	
+	@Test
+	public void testWaitForTermination() {
+		try {
+			SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+			latch.triggerNextSuperstep();
+			latch.triggerNextSuperstep();
+			
+			Waiter w = new Waiter(latch, 4);
+			Thread waiter = new Thread(w);
+			waiter.setDaemon(true);
+			waiter.start();
+			
+			WatchDog wd = new WatchDog(waiter, 2000);
+			wd.start();
+			
+			latch.signalTermination();
+			
+			wd.join();
+			if (wd.getError() != null) {
+				throw wd.getError();
+			}
+			
+			if (w.getError() != null) {
+				throw w.getError();
+			}
+		}
+		catch (Throwable t) {
+			t.printStackTrace();
+			Assert.fail("Error: " + t.getMessage());
+		}
+	}
+	
+	private static class Waiter implements Runnable {
+
+		private final SuperstepKickoffLatch latch;
+		
+		private final int waitFor;
+		
+		private volatile Throwable error;
+		
+		
+		public Waiter(SuperstepKickoffLatch latch, int waitFor) {
+			this.latch = latch;
+			this.waitFor = waitFor;
+		}
+
+		@Override
+		public void run() {
+			try {
+				latch.awaitStartOfSuperstepOrTermination(waitFor);
+			}
+			catch (Throwable t) {
+				this.error = t;
+			}
+		}
+		
+		public Throwable getError() {
+			return error;
+		}
+	}
+	
+	private static class WatchDog extends Thread {
+		
+		private final Thread toWatch;
+		
+		private final long timeOut;
+		
+		private volatile Throwable failed;
+		
+		public WatchDog(Thread toWatch, long timeout) {
+			setDaemon(true);
+			setName("Watchdog");
+			this.toWatch = toWatch;
+			this.timeOut = timeout;
+		}
+		
+		@SuppressWarnings("deprecation")
+		@Override
+		public void run() {
+			try {
+				toWatch.join(timeOut);
+				
+				if (toWatch.isAlive()) {
+					this.failed = new Exception("timed out");
+					toWatch.interrupt();
+					
+					toWatch.join(2000);
+					if (toWatch.isAlive()) {
+						toWatch.stop();
+					}
+				}
+			}
+			catch (Throwable t) {
+				failed = t;
+			}
+		}
+		
+		public Throwable getError() {
+			return failed;
+		}
+	}
+}


[3/3] git commit: Disable POJO types

Posted by rm...@apache.org.
Disable POJO types

There are various issues when working with POJOs in the Java API, for
example [1].

This commit squashes the following commmits:

1. Disable tests, which depend on POJO types

2. Replace expression keys with key selector in test

3. PackagedProgramEndToEndITCase used a KMeans variant, which relies on
expression keys to select the field to group on. This commit replaces
this with a key selector.

4. Disable WordCountPOJO test case and package

5. Disables WordCountPOJOITCase, which executes the WordCountPOJO example.
The example code is still included with the source code (with notice
that it is currently not working), but will *not* be packaged as a JAR
anymore.

6. Disable tests, which depend on POJO types (continued)

[1] https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E

This closes #93.


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/cb81fac8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/cb81fac8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/cb81fac8

Branch: refs/heads/release-0.6
Commit: cb81fac88b25d2dcadca32c11e88fba293d97700
Parents: 18f13ea
Author: uce <u....@fu-berlin.de>
Authored: Fri Aug 8 11:18:52 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Fri Aug 8 15:29:14 2014 +0200

----------------------------------------------------------------------
 .../io/AvroInputFormatTypeExtractionTest.java   | 38 ++++++++++----------
 flink-examples/flink-java-examples/pom.xml      |  8 +++--
 .../example/java/wordcount/WordCountPOJO.java   |  9 +++++
 .../flink/api/java/typeutils/TypeExtractor.java | 10 +++---
 .../api/java/operator/CoGroupOperatorTest.java  |  7 +++-
 .../flink/api/java/operator/GroupingTest.java   |  5 +++
 .../api/java/operator/JoinOperatorTest.java     |  5 +++
 .../type/extractor/PojoTypeInformationTest.java |  7 +++-
 .../java/type/extractor/TypeExtractorTest.java  | 12 +++----
 .../typeutils/runtime/PojoSerializerTest.java   | 32 +++++++++--------
 .../WordCountPOJOITCase.java                    |  3 +-
 .../PackagedProgramEndToEndITCase.java          |  1 -
 .../flink/test/util/testjar/KMeansForTest.java  |  8 ++++-
 13 files changed, 91 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
index aa08006..efe45fd 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
@@ -19,16 +19,15 @@
 
 package org.apache.flink.api.java.io;
 
-import org.junit.Assert;
-import org.junit.Test;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.AvroInputFormat;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.types.TypeInformation;
+import org.junit.Assert;
+import org.junit.Test;
 
 public class AvroInputFormatTypeExtractionTest {
 
@@ -36,44 +35,43 @@ public class AvroInputFormatTypeExtractionTest {
 	public void testTypeExtraction() {
 		try {
 			InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);
-			
+
 			TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);
-			
+
 			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 			DataSet<MyAvroType> input = env.createInput(format);
 			TypeInformation<?> typeInfoDataSet = input.getType();
-			
-			
-			Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
-			Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
-			
+
+
+			Assert.assertTrue(typeInfoDirect instanceof GenericTypeInfo);
+			Assert.assertTrue(typeInfoDataSet instanceof GenericTypeInfo);
+
 			Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
 			Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
-		}
-		catch (Exception e) {
+		} catch (Exception e) {
 			e.printStackTrace();
 			Assert.fail(e.getMessage());
 		}
 	}
-	
+
 	public static final class MyAvroType {
-		
+
 		public String theString;
-		
+
 		private double aDouble;
-		
+
 		public double getaDouble() {
 			return aDouble;
 		}
-		
+
 		public void setaDouble(double aDouble) {
 			this.aDouble = aDouble;
 		}
-		
+
 		public void setTheString(String theString) {
 			this.theString = theString;
 		}
-		
+
 		public String getTheString() {
 			return theString;
 		}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-examples/flink-java-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/pom.xml b/flink-examples/flink-java-examples/pom.xml
index ea0db5d..97ebdec 100644
--- a/flink-examples/flink-java-examples/pom.xml
+++ b/flink-examples/flink-java-examples/pom.xml
@@ -291,7 +291,10 @@ under the License.
 						</configuration>
 					</execution>
 
-					<!-- WordCountPOJO -->
+					<!-- WordCountPOJO
+
+					Note: disabled for now, because of problems with POJO types
+
 					<execution>
 						<id>WordCountPOJO</id>
 						<phase>package</phase>
@@ -313,8 +316,7 @@ under the License.
 								<include>**/java/wordcount/util/WordCountData.class</include>
 							</includes>
 						</configuration>
-					</execution>
-					
+					</execution> -->
 				</executions>
 			</plugin>
 		</plugins>

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
index 04810a1..d993e60 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
@@ -53,6 +53,15 @@ public class WordCountPOJO {
 	// *************************************************************************
 
 	public static void main(String[] args) throws Exception {
+		// ====================================================================
+		// IMPORTANT
+		//
+		// Note: this example is currently not working, because support for
+		// POJO types has been disabled. As soon as all known issues (see [1])
+		// are fixed, we will enable POJO support again.
+		//
+		// [1] https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E
+		// ====================================================================
 
 		parseParameters(args);
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index d03cc49..b596a87 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -792,11 +792,13 @@ public class TypeExtractor {
 			// special case handling for Class, this should not be handled by the POJO logic
 			return new GenericTypeInfo<X>(clazz);
 		}
-		TypeInformation<X> pojoType =  analyzePojo(clazz);
-		if (pojoType != null) {
-			return pojoType;
-		}
 
+//		Disable POJO types for now (see https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E)
+//
+//		TypeInformation<X> pojoType =  analyzePojo(clazz);
+//		if (pojoType != null) {
+//			return pojoType;
+//		}
 
 		// return a generic type
 		return new GenericTypeInfo<X>(clazz);

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
index 4bc7ac2..360050b 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.api.java.typeutils.BasicTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.flink.api.java.DataSet;
@@ -128,6 +129,7 @@ public class CoGroupOperatorTest {
 		ds1.coGroup(ds2).where(5).equalTo(0);
 	}
 
+	@Ignore
 	@Test
 	public void testCoGroupKeyExpressions1() {
 
@@ -143,6 +145,7 @@ public class CoGroupOperatorTest {
 		}
 	}
 
+	@Ignore
 	@Test(expected = InvalidProgramException.class)
 	public void testCoGroupKeyExpressions2() {
 
@@ -154,6 +157,7 @@ public class CoGroupOperatorTest {
 		ds1.coGroup(ds2).where("myInt").equalTo("myString");
 	}
 
+	@Ignore
 	@Test(expected = InvalidProgramException.class)
 	public void testCoGroupKeyExpressions3() {
 
@@ -165,6 +169,7 @@ public class CoGroupOperatorTest {
 		ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
 	}
 
+	@Ignore
 	@Test(expected = IllegalArgumentException.class)
 	public void testCoGroupKeyExpressions4() {
 
@@ -175,7 +180,7 @@ public class CoGroupOperatorTest {
 		// should not work, cogroup key non-existent
 		ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
 	}
-	
+
 	@Test
 	public void testCoGroupKeySelectors1() {
 		

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
index 0938bd9..98b6998 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.api.java.typeutils.BasicTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.flink.api.java.DataSet;
@@ -112,6 +113,7 @@ public class GroupingTest {
 		tupleDs.groupBy(-1);
 	}
 
+	@Ignore
 	@Test
 	public void testGroupByKeyExpressions1() {
 
@@ -129,6 +131,7 @@ public class GroupingTest {
 		}
 	}
 
+	@Ignore
 	@Test(expected = UnsupportedOperationException.class)
 	public void testGroupByKeyExpressions2() {
 
@@ -139,6 +142,7 @@ public class GroupingTest {
 		longDs.groupBy("myInt");
 	}
 
+	@Ignore
 	@Test(expected = InvalidProgramException.class)
 	public void testGroupByKeyExpressions3() {
 
@@ -152,6 +156,7 @@ public class GroupingTest {
 
 	}
 
+	@Ignore
 	@Test(expected = IllegalArgumentException.class)
 	public void testGroupByKeyExpressions4() {
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
index 8b88288..11eca83 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.api.java.typeutils.BasicTypeInfo;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.flink.api.java.DataSet;
@@ -128,6 +129,7 @@ public class JoinOperatorTest {
 		ds1.join(ds2).where(5).equalTo(0);
 	}
 
+	@Ignore
 	@Test
 	public void testJoinKeyExpressions1() {
 
@@ -143,6 +145,7 @@ public class JoinOperatorTest {
 		}
 	}
 
+	@Ignore
 	@Test(expected = InvalidProgramException.class)
 	public void testJoinKeyExpressions2() {
 
@@ -154,6 +157,7 @@ public class JoinOperatorTest {
 		ds1.join(ds2).where("myInt").equalTo("myString");
 	}
 
+	@Ignore
 	@Test(expected = InvalidProgramException.class)
 	public void testJoinKeyExpressions3() {
 
@@ -165,6 +169,7 @@ public class JoinOperatorTest {
 		ds1.join(ds2).where("myInt", "myString").equalTo("myString");
 	}
 
+	@Ignore
 	@Test(expected = IllegalArgumentException.class)
 	public void testJoinKeyExpressions4() {
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
index b963574..cba8e88 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
 import org.apache.flink.api.java.typeutils.PojoTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.types.TypeInformation;
+import org.junit.Ignore;
 import org.junit.Test;
 
 @SuppressWarnings("unused")
@@ -51,6 +52,7 @@ public class PojoTypeInformationTest {
 		Integer[] intWrapperArray;
 	}
 
+	@Ignore
 	@Test
 	public void testSimplePojoTypeExtraction() {
 		TypeInformation<SimplePojo> type = TypeExtractor.getForClass(SimplePojo.class);
@@ -66,6 +68,7 @@ public class PojoTypeInformationTest {
 		NestedPojoInner inner;
 	}
 
+	@Ignore
 	@Test
 	public void testNestedPojoTypeExtraction() {
 		TypeInformation<NestedPojoOuter> type = TypeExtractor.getForClass(NestedPojoOuter.class);
@@ -82,6 +85,7 @@ public class PojoTypeInformationTest {
 		Recursive1Pojo rec;
 	}
 
+	@Ignore
 	@Test
 	public void testRecursivePojoTypeExtraction() {
 		// This one tests whether a recursive pojo is detected using the set of visited
@@ -89,7 +93,8 @@ public class PojoTypeInformationTest {
 		TypeInformation<Recursive1Pojo> type = TypeExtractor.getForClass(Recursive1Pojo.class);
 		assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
 	}
-	
+
+	@Ignore
 	@Test
 	public void testRecursivePojoObjectTypeExtraction() {
 		TypeInformation<Recursive1Pojo> type = TypeExtractor.getForObject(new Recursive1Pojo());

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index c8a8ee9..412f751 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -41,7 +41,6 @@ import org.apache.flink.api.java.typeutils.BasicArrayTypeInfo;
 import org.apache.flink.api.java.typeutils.BasicTypeInfo;
 import org.apache.flink.api.java.typeutils.GenericTypeInfo;
 import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
 import org.apache.flink.api.java.typeutils.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -62,7 +61,6 @@ import org.junit.Test;
 
 public class TypeExtractorTest {
 
-	
 	@SuppressWarnings({ "rawtypes", "unchecked" })
 	@Test
 	public void testBasicType() {
@@ -307,11 +305,11 @@ public class TypeExtractorTest {
 
 		Assert.assertFalse(ti.isBasicType());
 		Assert.assertFalse(ti.isTupleType());
-		Assert.assertTrue(ti instanceof PojoTypeInfo);
+		Assert.assertTrue(ti instanceof GenericTypeInfo);
 		Assert.assertEquals(ti.getTypeClass(), CustomType.class);
 
 		// use getForClass()
-		Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof PojoTypeInfo);
+		Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof GenericTypeInfo);
 		Assert.assertEquals(TypeExtractor.getForClass(CustomType.class).getTypeClass(), ti.getTypeClass());
 
 		// use getForObject()
@@ -320,7 +318,7 @@ public class TypeExtractorTest {
 
 		Assert.assertFalse(ti2.isBasicType());
 		Assert.assertFalse(ti2.isTupleType());
-		Assert.assertTrue(ti2 instanceof PojoTypeInfo);
+		Assert.assertTrue(ti2 instanceof GenericTypeInfo);
 		Assert.assertEquals(ti2.getTypeClass(), CustomType.class);
 	}
 
@@ -360,7 +358,7 @@ public class TypeExtractorTest {
 		Assert.assertEquals(Tuple2.class, tti.getTypeClass());
 		
 		Assert.assertEquals(Long.class, tti.getTypeAt(0).getTypeClass());
-		Assert.assertTrue(tti.getTypeAt(1) instanceof PojoTypeInfo);
+		Assert.assertTrue(tti.getTypeAt(1) instanceof GenericTypeInfo);
 		Assert.assertEquals(CustomType.class, tti.getTypeAt(1).getTypeClass());
 
 		// use getForObject()
@@ -373,7 +371,7 @@ public class TypeExtractorTest {
 		
 		Assert.assertEquals(Tuple2.class, tti2.getTypeClass());
 		Assert.assertEquals(Long.class, tti2.getTypeAt(0).getTypeClass());
-		Assert.assertTrue(tti2.getTypeAt(1) instanceof PojoTypeInfo);
+		Assert.assertTrue(tti2.getTypeAt(1) instanceof GenericTypeInfo);
 		Assert.assertEquals(CustomType.class, tti2.getTypeAt(1).getTypeClass());
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index 5aef84e..2d17764 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -18,48 +18,48 @@
 
 package org.apache.flink.api.java.typeutils.runtime;
 
-import java.util.Random;
-
+import com.google.common.base.Objects;
 import org.apache.flink.api.common.typeutils.SerializerTestBase;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
 import org.apache.flink.types.TypeInformation;
+import org.junit.Ignore;
 
-import com.google.common.base.Objects;
+import java.util.Random;
 
 /**
  * A test for the {@link org.apache.flink.api.java.typeutils.runtime.PojoSerializer}.
  */
+@Ignore
 public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.TestUserClass> {
 	private TypeInformation<TestUserClass> type = TypeExtractor.getForClass(TestUserClass.class);
 
 	@Override
 	protected TypeSerializer<TestUserClass> createSerializer() {
 		TypeSerializer<TestUserClass> serializer = type.createSerializer();
-		assert(serializer instanceof PojoSerializer);
+		assert (serializer instanceof PojoSerializer);
 		return serializer;
 	}
-	
+
 	@Override
 	protected int getLength() {
 		return -1;
 	}
-	
+
 	@Override
 	protected Class<TestUserClass> getTypeClass() {
 		return TestUserClass.class;
 	}
-	
+
 	@Override
 	protected TestUserClass[] getTestData() {
 		Random rnd = new Random(874597969123412341L);
 
-		return new TestUserClass[] {
-			new TestUserClass(rnd.nextInt(), "foo", rnd.nextDouble(), new int[] {1,2,3},
-					new NestedTestUserClass(rnd.nextInt(), "foo@boo", rnd.nextDouble(), new int[] {10, 11, 12})),
-			new TestUserClass(rnd.nextInt(), "bar", rnd.nextDouble(), new int[] {4,5,6},
-					new NestedTestUserClass(rnd.nextInt(), "bar@bas", rnd.nextDouble(), new int[] {20, 21, 22}))
+		return new TestUserClass[]{
+				new TestUserClass(rnd.nextInt(), "foo", rnd.nextDouble(), new int[]{1, 2, 3},
+						new NestedTestUserClass(rnd.nextInt(), "foo@boo", rnd.nextDouble(), new int[]{10, 11, 12})),
+				new TestUserClass(rnd.nextInt(), "bar", rnd.nextDouble(), new int[]{4, 5, 6},
+						new NestedTestUserClass(rnd.nextInt(), "bar@bas", rnd.nextDouble(), new int[]{20, 21, 22}))
 		};
 
 	}
@@ -73,7 +73,8 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
 
 		private NestedTestUserClass nestedClass;
 
-		public TestUserClass() {}
+		public TestUserClass() {
+		}
 
 		public TestUserClass(int dumm1, String dumm2, double dumm3, int[] dumm4, NestedTestUserClass nestedClass) {
 			this.dumm1 = dumm1;
@@ -124,7 +125,8 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
 		public double dumm3;
 		private int[] dumm4;
 
-		public NestedTestUserClass() {}
+		public NestedTestUserClass() {
+		}
 
 		public NestedTestUserClass(int dumm1, String dumm2, double dumm3, int[] dumm4) {
 			this.dumm1 = dumm1;

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
index f463924..7e16fd4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
@@ -21,8 +21,9 @@ package org.apache.flink.test.exampleJavaPrograms;
 import org.apache.flink.example.java.wordcount.WordCountPOJO;
 import org.apache.flink.test.testdata.WordCountData;
 import org.apache.flink.test.util.JavaProgramTestBase;
+import org.junit.Ignore;
 
-
+@Ignore
 public class WordCountPOJOITCase extends JavaProgramTestBase {
 
 	protected String textPath;

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java b/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
index 0e93f2a..d244811 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
@@ -64,7 +64,6 @@ public class PackagedProgramEndToEndITCase {
 			fwClusters.close();
 
 			String jarPath = "target/maven-test-jar.jar";
-//			String jarPath = "/home/aljoscha/maven-test-jar.jar";
 
 			// run KMeans
 			cluster.setNumTaskTracker(2);

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cb81fac8/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
index 7149cd3..917615d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
@@ -24,6 +24,7 @@ import java.util.Collection;
 
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.api.common.Program;
+import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.functions.RichMapFunction;
 import org.apache.flink.api.java.functions.RichReduceFunction;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -79,7 +80,12 @@ public class KMeansForTest implements Program {
 			.map(new SelectNearestCenter()).withBroadcastSet(loop, "centroids")
 			// count and sum point coordinates for each centroid
 			.map(new CountAppender())
-			.groupBy("f0").reduce(new CentroidAccumulator())
+			.groupBy(new KeySelector<DummyTuple3IntPointLong, Integer>() {
+				@Override
+				public Integer getKey(DummyTuple3IntPointLong value) throws Exception {
+					return value.f0;
+				}
+			}).reduce(new CentroidAccumulator())
 			// compute new centroids from point counts and coordinate sums
 			.map(new CentroidAverager());