You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/07/03 14:51:02 UTC

[1/3] git commit: [FLINK-996] Fix for bug in union replacement

Repository: incubator-flink
Updated Branches:
  refs/heads/master 8a8060eaa -> 32a003d5c


[FLINK-996] Fix for bug in union replacement

This closes #56


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/32a003d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/32a003d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/32a003d5

Branch: refs/heads/master
Commit: 32a003d5c0f507f4634851434e8ffb8ce6b880b2
Parents: cf29798
Author: Till Rohrmann <ti...@gmail.com>
Authored: Wed Jul 2 16:00:55 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200

----------------------------------------------------------------------
 .../eu/stratosphere/compiler/PactCompiler.java  | 53 ++++++--------------
 .../pact/compiler/UnionReplacementTest.java     | 49 ++++++++++++++++++
 2 files changed, 64 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/32a003d5/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
----------------------------------------------------------------------
diff --git a/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java b/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
index bf3d6af..5a0a3e1 100644
--- a/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
+++ b/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
@@ -1195,7 +1195,7 @@ public class PactCompiler {
 	private static final class BinaryUnionReplacer implements Visitor<PlanNode> {
 		
 		private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
-		
+
 		@Override
 		public boolean preVisit(PlanNode visitable) {
 			if (this.seenBefore.add(visitable)) {
@@ -1217,43 +1217,20 @@ public class PactCompiler {
 				final Channel in2 = unionNode.getInput2();
 			
 				PlanNode newUnionNode;
-				
-				// if any input is cached, we keep this as a binary union and do not collapse it into a
-				// n-ary union
-//				if (in1.getTempMode().isCached() || in2.getTempMode().isCached()) {
-//					// replace this node by an explicit operator
-//					Channel cached, pipelined;
-//					if (in1.getTempMode().isCached()) {
-//						cached = in1;
-//						pipelined = in2;
-//					} else {
-//						cached = in2;
-//						pipelined = in1;
-//					}
-//					
-//					newUnionNode = new DualInputPlanNode(unionNode.getOriginalOptimizerNode(), cached, pipelined,
-//						DriverStrategy.UNION_WITH_CACHED);
-//					newUnionNode.initProperties(unionNode.getGlobalProperties(), new LocalProperties());
-//					
-//					in1.setTarget(newUnionNode);
-//					in2.setTarget(newUnionNode);
-//				} else {
-					// collect the union inputs to collapse this operator with 
-					// its collapsed predecessors. check whether an input is materialized to prevent
-					// collapsing
-					List<Channel> inputs = new ArrayList<Channel>();
-					collect(in1, inputs);
-					collect(in2, inputs);
-					
-					newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties());
-					
-					// adjust the input channels to have their target point to the new union node
-					for (Channel c : inputs) {
-						c.setTarget(newUnionNode);
-					}
-//				}
-				
-				unionNode.getOutgoingChannels().get(0).swapUnionNodes(newUnionNode);
+
+				List<Channel> inputs = new ArrayList<Channel>();
+				collect(in1, inputs);
+				collect(in2, inputs);
+
+				newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties());
+
+				for (Channel c : inputs) {
+					c.setTarget(newUnionNode);
+				}
+
+				for(Channel channel : unionNode.getOutgoingChannels()){
+					channel.swapUnionNodes(newUnionNode);
+				}
 			}
 		}
 		

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/32a003d5/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
----------------------------------------------------------------------
diff --git a/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java b/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
new file mode 100644
index 0000000..9ef8519
--- /dev/null
+++ b/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package eu.stratosphere.pact.compiler;
+
+import eu.stratosphere.api.common.Plan;
+import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.ExecutionEnvironment;
+import eu.stratosphere.api.java.IterativeDataSet;
+import eu.stratosphere.compiler.CompilerException;
+import eu.stratosphere.compiler.plan.OptimizedPlan;
+import eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator;
+import org.junit.Test;
+import static org.junit.Assert.fail;
+
+public class UnionReplacementTest extends CompilerTestBase {
+
+	@Test
+	public void testUnionReplacement(){
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<String> input1 = env.fromElements("test1");
+		DataSet<String> input2 = env.fromElements("test2");
+
+		DataSet<String> union = input1.union(input2);
+
+		union.print();
+		union.print();
+
+		Plan plan = env.createProgramPlan();
+		try{
+			OptimizedPlan oPlan = this.compileNoStats(plan);
+			NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+			jobGen.compileJobGraph(oPlan);
+		}catch(CompilerException co){
+			co.printStackTrace();
+			fail("The Pact compiler is unable to compile this plan correctly.");
+		}
+	}
+}


[3/3] git commit: [FLINK-932] Introduce automatic generation of semantic props for projections

Posted by se...@apache.org.
[FLINK-932] Introduce automatic generation of semantic props for projections

This closes #29


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/883474dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/883474dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/883474dd

Branch: refs/heads/master
Commit: 883474ddfd749eedceb85bdd2925a3a76de75206
Parents: 8a8060e
Author: sebastian kunert <sk...@gmail.com>
Authored: Thu Jun 19 16:17:59 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200

----------------------------------------------------------------------
 .../api/java/functions/SemanticPropUtil.java    |  21 +++
 .../api/java/operators/CrossOperator.java       |  34 ++++-
 .../api/java/operators/JoinOperator.java        |  36 ++++-
 .../api/java/operators/ProjectOperator.java     |   5 +-
 .../SemanticPropertiesProjectionTest.java       | 144 +++++++++++++++++++
 5 files changed, 233 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
index 4d767b0..9f661f0 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
@@ -48,6 +48,27 @@ public class SemanticPropUtil {
 
 	private static final Pattern PATTERN_DIGIT = Pattern.compile("\\d+");
 
+	public static SingleInputSemanticProperties createProjectionPropertiesSingle(int[] fields) {
+		SingleInputSemanticProperties ssp = new SingleInputSemanticProperties();
+		for (int i = 0; i < fields.length; i++) {
+			ssp.addForwardedField(fields[i], i);
+		}
+		return ssp;
+	}
+
+	public static DualInputSemanticProperties createProjectionPropertiesDual(int[] fields, boolean[] isFromFirst) {
+		DualInputSemanticProperties dsp = new DualInputSemanticProperties();
+
+		for (int i = 0; i < fields.length; i++) {
+			if (isFromFirst[i]) {
+				dsp.addForwardedField1(fields[i], i);
+			} else {
+				dsp.addForwardedField2(fields[i], i);
+			}
+		}
+		return dsp;
+	}
+
 	public static SingleInputSemanticProperties getSemanticPropsSingle(Set<Annotation> set, TypeInformation<?> inType, TypeInformation<?> outType) {
 		if (set == null) {
 			return null;

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
index b497407..76e5320 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
@@ -18,10 +18,12 @@ import java.util.Arrays;
 
 import eu.stratosphere.api.common.functions.GenericCrosser;
 import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
 import eu.stratosphere.api.common.operators.Operator;
 import eu.stratosphere.api.common.operators.base.CrossOperatorBase;
 import eu.stratosphere.api.java.DataSet;
 import eu.stratosphere.api.java.functions.CrossFunction;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
 import eu.stratosphere.api.java.typeutils.TupleTypeInfo;
 import eu.stratosphere.api.java.typeutils.TypeExtractor;
 //CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
@@ -49,9 +51,19 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		super(input1, input2, returnType);
 
 		this.function = function;
-		extractSemanticAnnotationsFromUdf(function.getClass());
+
+		if (!(function instanceof ProjectCrossFunction)) {
+			extractSemanticAnnotationsFromUdf(function.getClass());
+		} else {
+			generateProjectionProperties(((ProjectCrossFunction) function));
+		}
+	}
+
+	public void generateProjectionProperties(ProjectCrossFunction pcf) {
+		DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pcf.getFields(), pcf.getIsFromFirst());
+		setSemanticProperties(props);
 	}
-	
+
 	@Override
 	protected eu.stratosphere.api.common.operators.base.CrossOperatorBase<I1, I2, OUT, GenericCrosser<I1,I2,OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
 		
@@ -187,6 +199,16 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 			super(input1, input2,
 				new ProjectCrossFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()), returnType);
 		}
+
+		@Override
+		public CrossOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
+			throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+		}
+
+		@Override
+		public CrossOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
+			throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+		}
 	}
 
 	public static final class ProjectCrossFunction<T1, T2, R extends Tuple> extends CrossFunction<T1, T2, R> {
@@ -234,6 +256,14 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 			}
 			return outTuple;
 		}
+
+		protected int[] getFields() {
+			return fields;
+		}
+
+		protected boolean[] getIsFromFirst() {
+			return isFromFirst;
+		}
 	}
 
 	public static final class CrossProjection<I1, I2> {

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
index 9479c28..c96f05c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
@@ -21,6 +21,7 @@ import eu.stratosphere.api.common.InvalidProgramException;
 import eu.stratosphere.api.common.functions.GenericJoiner;
 import eu.stratosphere.api.common.functions.GenericMap;
 import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
 import eu.stratosphere.api.common.operators.Operator;
 import eu.stratosphere.api.common.operators.UnaryOperatorInformation;
 import eu.stratosphere.api.common.operators.base.JoinOperatorBase;
@@ -29,6 +30,7 @@ import eu.stratosphere.api.java.DataSet;
 import eu.stratosphere.api.java.DeltaIteration.SolutionSetPlaceHolder;
 import eu.stratosphere.api.java.functions.JoinFunction;
 import eu.stratosphere.api.java.functions.KeySelector;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
 import eu.stratosphere.api.java.operators.Keys.FieldPositionKeys;
 import eu.stratosphere.api.java.operators.translation.KeyExtractingMapper;
 import eu.stratosphere.api.java.operators.translation.PlanUnwrappingJoinOperator;
@@ -163,9 +165,19 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			}
 			
 			this.function = function;
-			extractSemanticAnnotationsFromUdf(function.getClass());
+
+			if (!(function instanceof ProjectJoinFunction)) {
+				extractSemanticAnnotationsFromUdf(function.getClass());
+			} else {
+				generateProjectionProperties(((ProjectJoinFunction) function));
+			}
 		}
-		
+
+		public void generateProjectionProperties(ProjectJoinFunction pjf) {
+			DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pjf.getFields(), pjf.getIsFromFirst());
+			setSemanticProperties(props);
+		}
+
 		// TODO
 //		public EquiJoin<I1, I2, OUT> leftOuter() {
 //			this.preserve1 = true;
@@ -512,6 +524,16 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 					new ProjectJoinFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()), 
 					returnType, hint);
 		}
+
+		@Override
+		public JoinOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
+			throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+		}
+
+		@Override
+		public JoinOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
+			throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+		}
 	}
 	
 //	@SuppressWarnings("unused")
@@ -824,7 +846,15 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			this.isFromFirst = isFromFirst;
 			this.outTuple = outTupleInstance;
 		}
-		
+
+		protected int[] getFields() {
+			return fields;
+		}
+
+		protected boolean[] getIsFromFirst() {
+			return isFromFirst;
+		}
+
 		public R join(T1 in1, T2 in2) {
 			for(int i=0; i<fields.length; i++) {
 				if(isFromFirst[i]) {

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
index 1907393..35406d6 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
@@ -19,6 +19,7 @@ import java.util.Arrays;
 import eu.stratosphere.api.common.functions.GenericMap;
 import eu.stratosphere.api.common.operators.Operator;
 import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
 import eu.stratosphere.api.java.operators.translation.PlanProjectOperator;
 //CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
 import eu.stratosphere.api.java.tuple.*;
@@ -54,11 +55,11 @@ public class ProjectOperator<IN, OUT extends Tuple>
 		ppo.setInput(input);
 		// set dop
 		ppo.setDegreeOfParallelism(this.getParallelism());
-		
+		ppo.setSemanticProperties(SemanticPropUtil.createProjectionPropertiesSingle(fields));
+
 		return ppo;
 	}
 
-	
 	public static class Projection<T> {
 		
 		private final DataSet<T> ds;

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java b/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
new file mode 100644
index 0000000..4e812da
--- /dev/null
+++ b/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
@@ -0,0 +1,144 @@
+/***********************************************************************************************************************
+ *
+ * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ **********************************************************************************************************************/
+
+package eu.stratosphere.api.java.functions;
+
+import eu.stratosphere.api.common.Plan;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
+import eu.stratosphere.api.common.operators.SingleInputSemanticProperties;
+import eu.stratosphere.api.common.operators.base.CrossOperatorBase;
+import eu.stratosphere.api.common.operators.base.GenericDataSinkBase;
+import eu.stratosphere.api.common.operators.base.JoinOperatorBase;
+import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.ExecutionEnvironment;
+import eu.stratosphere.api.java.operators.translation.PlanProjectOperator;
+import eu.stratosphere.api.java.tuple.Tuple4;
+import eu.stratosphere.api.java.tuple.Tuple5;
+import eu.stratosphere.api.java.typeutils.BasicTypeInfo;
+import eu.stratosphere.api.java.typeutils.TupleTypeInfo;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Created by sebastian on 6/19/14.
+ */
+public class SemanticPropertiesProjectionTest {
+
+	final List<Tuple5<Integer, Long, String, Long, Integer>> emptyTupleData = new ArrayList<Tuple5<Integer, Long, String, Long, Integer>>();
+
+	final TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>> tupleTypeInfo = new
+			TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>>(
+			BasicTypeInfo.INT_TYPE_INFO,
+			BasicTypeInfo.LONG_TYPE_INFO,
+			BasicTypeInfo.STRING_TYPE_INFO,
+			BasicTypeInfo.LONG_TYPE_INFO,
+			BasicTypeInfo.INT_TYPE_INFO
+	);
+
+
+	@Test
+	public void ProjectOperatorTest() {
+		try {
+			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+			DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+			tupleDs.project(1, 3, 2).types(Long.class, Long.class, String.class).print();
+
+			Plan plan = env.createProgramPlan();
+
+			GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+			PlanProjectOperator<?, ?> projectOperator = ((PlanProjectOperator) sink.getInput());
+
+			SingleInputSemanticProperties props = projectOperator.getSemanticProperties();
+
+			assertTrue(props.getForwardedField(1).size() == 1);
+			assertTrue(props.getForwardedField(3).size() == 1);
+			assertTrue(props.getForwardedField(2).size() == 1);
+			assertTrue(props.getForwardedField(1).contains(0));
+			assertTrue(props.getForwardedField(3).contains(1));
+			assertTrue(props.getForwardedField(2).contains(2));
+		} catch (Exception e) {
+			System.err.println(e.getMessage());
+			e.printStackTrace();
+			fail("Exception in test: " + e.getMessage());
+		}
+	}
+
+	@Test
+	public void JoinProjectionTest() {
+		try {
+			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+			DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+			tupleDs.join(tupleDs).where(0).equalTo(0).projectFirst(2, 3).projectSecond(1, 4).types(String.class, Long.class, Long.class, Integer.class).print();
+
+			Plan plan = env.createProgramPlan();
+
+			GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+			JoinOperatorBase<?, ?, ?, ?> projectJoinOperator = ((JoinOperatorBase<?, ?, ?, ?>) sink.getInput());
+
+			DualInputSemanticProperties props = projectJoinOperator.getSemanticProperties();
+
+			assertTrue(props.getForwardedField1(2).size() == 1);
+			assertTrue(props.getForwardedField1(3).size() == 1);
+			assertTrue(props.getForwardedField2(1).size() == 1);
+			assertTrue(props.getForwardedField2(4).size() == 1);
+			assertTrue(props.getForwardedField1(2).contains(0));
+			assertTrue(props.getForwardedField1(3).contains(1));
+			assertTrue(props.getForwardedField2(1).contains(2));
+			assertTrue(props.getForwardedField2(4).contains(3));
+		} catch (Exception e) {
+			System.err.println(e.getMessage());
+			e.printStackTrace();
+			fail("Exception in test: " + e.getMessage());
+		}
+	}
+
+	@Test
+	public void CrossProjectionTest() {
+		try {
+			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+			DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+			DataSet<Tuple4<String, Long, Long, Integer>> result = tupleDs.cross(tupleDs).projectFirst(2, 3).projectSecond(1, 4).types(String.class, Long.class, Long.class, Integer.class);
+			result.print();
+
+			Plan plan = env.createProgramPlan();
+
+			GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+			CrossOperatorBase<?, ?, ?, ?> projectCrossOperator = ((CrossOperatorBase<?, ?, ?, ?>) sink.getInput());
+
+			DualInputSemanticProperties props = projectCrossOperator.getSemanticProperties();
+
+			assertTrue(props.getForwardedField1(2).size() == 1);
+			assertTrue(props.getForwardedField1(3).size() == 1);
+			assertTrue(props.getForwardedField2(1).size() == 1);
+			assertTrue(props.getForwardedField2(4).size() == 1);
+			assertTrue(props.getForwardedField1(2).contains(0));
+			assertTrue(props.getForwardedField1(3).contains(1));
+			assertTrue(props.getForwardedField2(1).contains(2));
+			assertTrue(props.getForwardedField2(4).contains(3));
+		} catch (Exception e) {
+			System.err.println(e.getMessage());
+			e.printStackTrace();
+			fail("Exception in test: " + e.getMessage());
+		}
+	}
+}


[2/3] git commit: Fix various error messages and invalid JavaDocs.

Posted by se...@apache.org.
Fix various error messages and invalid JavaDocs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/cf29798b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/cf29798b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/cf29798b

Branch: refs/heads/master
Commit: cf29798b20572db29051c0c67658e18ac92e998e
Parents: 883474d
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 3 13:33:53 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200

----------------------------------------------------------------------
 .../api/java/operators/CrossOperator.java       | 52 ++++++++------
 .../api/java/operators/JoinOperator.java        | 73 +++++++++++---------
 .../java/operators/SingleInputUdfOperator.java  |  6 +-
 .../api/java/operators/TwoInputUdfOperator.java |  9 ++-
 4 files changed, 79 insertions(+), 61 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
index 76e5320..1b0bc0c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
@@ -16,6 +16,7 @@ package eu.stratosphere.api.java.operators;
 
 import java.util.Arrays;
 
+import eu.stratosphere.api.common.InvalidProgramException;
 import eu.stratosphere.api.common.functions.GenericCrosser;
 import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
 import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
@@ -55,11 +56,11 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		if (!(function instanceof ProjectCrossFunction)) {
 			extractSemanticAnnotationsFromUdf(function.getClass());
 		} else {
-			generateProjectionProperties(((ProjectCrossFunction) function));
+			generateProjectionProperties(((ProjectCrossFunction<?, ?, ?>) function));
 		}
 	}
 
-	public void generateProjectionProperties(ProjectCrossFunction pcf) {
+	public void generateProjectionProperties(ProjectCrossFunction<?, ?, ?> pcf) {
 		DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pcf.getFields(), pcf.getIsFromFirst());
 		setSemanticProperties(props);
 	}
@@ -92,7 +93,6 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 	 * 
 	 * @param <I1> The type of the first input DataSet of the Cross transformation.
 	 * @param <I2> The type of the second input DataSet of the Cross transformation.
-	 * @param <OUT> The type of the result of the Cross transformation.
 	 * 
 	 * @see Tuple2
 	 * @see DataSet
@@ -140,18 +140,20 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 * If the first cross input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 *
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
 		 *
-		 * @param fieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
+		 * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
-		 * @return A CrossProjection that needs to be converted into a {@link ProjectCross} to complete the
+		 * @return A CrossProjection that needs to be converted into a 
+		 *           {@link eu.stratosphere.api.java.operators.CrossOperator.ProjectCross} to complete the
 		 *           Cross transformation by calling {@link CrossProjection#types()}.
 		 *
 		 * @see Tuple
 		 * @see DataSet
-		 * @see CrossProjection
-		 * @see ProjectCross
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
 		 */
 		public CrossProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
 			return new CrossProjection<I1, I2>(input1, input2, firstFieldIndexes, null);
@@ -163,18 +165,20 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 * If the second cross input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 *
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
 		 *
-		 * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+		 * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
-		 * @return A CrossProjection that needs to be converted into a {@link ProjectCross} to complete the
+		 * @return A CrossProjection that needs to be converted into a
+		 *           {@link eu.stratosphere.api.java.operators.CrossOperator.ProjectCross} to complete the
 		 *           Cross transformation by calling {@link CrossProjection#types()}.
 		 *
 		 * @see Tuple
 		 * @see DataSet
-		 * @see CrossProjection
-		 * @see ProjectCross
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
 		 */
 		public CrossProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
 			return new CrossProjection<I1, I2>(input1, input2, null, secondFieldIndexes);
@@ -202,12 +206,12 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 
 		@Override
 		public CrossOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
-			throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+			throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
 		}
 
 		@Override
 		public CrossOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
-			throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+			throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
 		}
 	}
 
@@ -364,9 +368,10 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 * If the first cross input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 *
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
 		 *
-		 * @param fieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
+		 * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
 		 * @return A CrossProjection that needs to be converted into a {@link ProjectOperator} to complete the
@@ -374,8 +379,8 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 *
 		 * @see Tuple
 		 * @see DataSet
-		 * @see CrossProjection
-		 * @see ProjectCross
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
 		 */
 		public CrossProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
 
@@ -431,9 +436,10 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 * If the second cross input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 *
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
 		 *
-		 * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+		 * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
 		 * @return A CrossProjection that needs to be converted into a {@link ProjectOperator} to complete the
@@ -441,8 +447,8 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
 		 *
 		 * @see Tuple
 		 * @see DataSet
-		 * @see CrossProjection
-		 * @see ProjectCross
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+		 * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
 		 */
 		public CrossProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
index c96f05c..509585c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
@@ -169,11 +169,11 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			if (!(function instanceof ProjectJoinFunction)) {
 				extractSemanticAnnotationsFromUdf(function.getClass());
 			} else {
-				generateProjectionProperties(((ProjectJoinFunction) function));
+				generateProjectionProperties(((ProjectJoinFunction<?, ?, ?>) function));
 			}
 		}
 
-		public void generateProjectionProperties(ProjectJoinFunction pjf) {
+		public void generateProjectionProperties(ProjectJoinFunction<?, ?, ?> pjf) {
 			DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pjf.getFields(), pjf.getIsFromFirst());
 			setSemanticProperties(props);
 		}
@@ -431,7 +431,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * @return An EquiJoin that represents the joined result DataSet
 		 * 
 		 * @see JoinFunction
-		 * @see EquiJoin
+		 * @see eu.stratosphere.api.java.operators.JoinOperator.EquiJoin
 		 * @see DataSet
 		 */
 		public <R> EquiJoin<I1, I2, R> with(JoinFunction<I1, I2, R> function) {
@@ -448,18 +448,20 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * If the first join input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 * 
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
 		 * 
 		 * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
-		 * @return A JoinProjection that needs to be converted into a {@link ProjectJoin} to complete the 
-		 *           Join transformation by calling {@link JoinProjection#types()}.
+		 * @return A JoinProjection that needs to be converted into a
+		 *           {@link eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin} to complete the 
+		 *           Join transformation by calling the corresponding {@code types()} function.
 		 * 
 		 * @see Tuple
 		 * @see DataSet
-		 * @see JoinProjection
-		 * @see ProjectJoin
+		 * @see eu.stratosphere.api.java.operators.JoinOperator.JoinProjection
+		 * @see eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin
 		 */
 		public JoinProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
 			return new JoinProjection<I1, I2>(getInput1(), getInput2(), getKeys1(), getKeys2(), getJoinHint(), firstFieldIndexes, null);
@@ -471,18 +473,20 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * If the second join input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 * 
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
 		 * 
-		 * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields. 
+		 * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields. 
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
-		 * @return A JoinProjection that needs to be converted into a {@link ProjectJoin} to complete the 
-		 *           Join transformation by calling {@link JoinProjection#types()}.
+		 * @return A JoinProjection that needs to be converted into a
+		 *           {@link eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin} to complete the 
+		 *           Join transformation by calling the corresponding {@code types()} function.
 		 * 
 		 * @see Tuple
 		 * @see DataSet
-		 * @see JoinProjection
-		 * @see ProjectJoin
+		 * @see eu.stratosphere.api.java.operators.JoinOperator.JoinProjection
+		 * @see eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin
 		 */
 		public JoinProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
 			return new JoinProjection<I1, I2>(getInput1(), getInput2(), getKeys1(), getKeys2(), getJoinHint(), null, secondFieldIndexes);
@@ -527,12 +531,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 
 		@Override
 		public JoinOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
-			throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+			throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
 		}
 
 		@Override
 		public JoinOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
-			throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+			throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
 		}
 	}
 	
@@ -601,7 +605,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 	/**
 	 * Intermediate step of a Join transformation. <br/>
 	 * To continue the Join transformation, select the join key of the first input {@link DataSet} by calling 
-	 * {@link JoinOperatorSets#where(int...)} or {@link JoinOperatorSets#where(KeySelector)}.
+	 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets#where(int, int...)} or
+	 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets#where(KeySelector)}.
 	 *
 	 * @param <I1> The type of the first input DataSet of the Join transformation.
 	 * @param <I2> The type of the second input DataSet of the Join transformation.
@@ -635,7 +640,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * @param field0 The first index of the Tuple fields of the first join DataSets that should be used as key
 		 * @param fields The indexes of the other Tuple fields of the first join DataSets that should be used as keys.
 		 * @return An incomplete Join transformation. 
-		 *           Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+		 *           Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+		 *           {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
 		 *           to continue the Join. 
 		 * 
 		 * @see Tuple
@@ -656,7 +662,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * @param field0 The first field of the Tuple fields of the first join DataSets that should be used as key
 		 * @param fields The  fields of the first join DataSets that should be used as keys.
 		 * @return An incomplete Join transformation.
-		 *           Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+		 *           Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+		 *           {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
 		 *           to continue the Join.
 		 *
 		 * @see Tuple
@@ -676,7 +683,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * 
 		 * @param keySelector The KeySelector function which extracts the key values from the DataSet on which it is joined.
 		 * @return An incomplete Join transformation. 
-		 *           Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+		 *           Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+		 *           {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
 		 *           to continue the Join. 
 		 * 
 		 * @see KeySelector
@@ -691,7 +699,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		/**
 		 * Intermediate step of a Join transformation. <br/>
 		 * To continue the Join transformation, select the join key of the second input {@link DataSet} by calling 
-		 * {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}.
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or 
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}.
 		 *
 		 */
 		public class JoinOperatorSetsPredicate {
@@ -715,7 +724,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			 * {@link DataSet} that should be used as join keys.<br/>
 			 * <b>Note: Fields can only be selected as join keys on Tuple DataSets.</b><br/>
 			 * 
-			 * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with 
+			 * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with 
 			 * the element of the first input being the first field of the tuple and the element of the 
 			 * second input being the second field of the tuple. 
 			 *
@@ -734,7 +743,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			 * Continues a Join transformation and defines the  fields of the second join
 			 * {@link DataSet} that should be used as join keys.<br/>
 			 *
-			 * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
+			 * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
 			 * the element of the first input being the first field of the tuple and the element of the
 			 * second input being the second field of the tuple.
 			 *
@@ -754,7 +763,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 			 * The KeySelector function is called for each element of the second DataSet and extracts a single 
 			 * key value on which the DataSet is joined. </br>
 			 * 
-			 * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with 
+			 * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with 
 			 * the element of the first input being the first field of the tuple and the element of the 
 			 * second input being the second field of the tuple. 
 			 * 
@@ -999,18 +1008,17 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * If the first join input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 * 
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and 
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
 		 * 
 		 * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
 		 * @return A JoinProjection that needs to be converted into a {@link ProjectOperator} to complete the 
-		 *           ProjectJoin transformation by calling {@link JoinProjection#types()}.
+		 *           ProjectJoin transformation by calling the corresponding {@code types()} function.
 		 * 
 		 * @see Tuple
 		 * @see DataSet
-		 * @see JoinProjection
-		 * @see ProjectJoin
 		 */
 		public JoinProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
 			
@@ -1066,18 +1074,17 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
 		 * If the second join input is not a Tuple DataSet, no parameters should be passed.<br/>
 		 * 
 		 * Fields of the first and second input can be added by chaining the method calls of
-		 * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+		 * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
 		 * 
-		 * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields. 
+		 * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields. 
 		 * 					   For a non-Tuple DataSet, do not provide parameters.
 		 * 					   The order of fields in the output tuple is defined by to the order of field indexes.
 		 * @return A JoinProjection that needs to be converted into a {@link ProjectOperator} to complete the 
-		 *           ProjectJoin transformation by calling {@link JoinProjection#types()}.
+		 *           ProjectJoin transformation by calling the corresponding {@code types()} function.
 		 * 
 		 * @see Tuple
 		 * @see DataSet
-		 * @see JoinProjection
-		 * @see ProjectJoin
 		 */
 		public JoinProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
 			

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
index 75e8cb5..4e9c567 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
@@ -30,7 +30,8 @@ import eu.stratosphere.configuration.Configuration;
 /**
  * The <tt>SingleInputUdfOperator</tt> is the base class of all unary operators that execute
  * user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have one input (such as {@link MapFunction} or {@link ReduceFunction}).
+ * have one input (such as {@link eu.stratosphere.api.java.functions.MapFunction} or
+ * {@link eu.stratosphere.api.java.functions.ReduceFunction}).
  * <p>
  * This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
  * through configuration objects, and semantic properties.
@@ -98,7 +99,8 @@ public abstract class SingleInputUdfOperator<IN, OUT, O extends SingleInputUdfOp
 	 * <p>
 	 * Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
 	 * In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
-	 * Constant set annotations can only be specified if the second input and the output type of the UDF are of {@link Tuple} data types.
+	 * Constant set annotations can only be specified if the second input and the output type of the UDF are of
+	 * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
 	 * 
 	 * <p>
 	 * A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
index ef622de..c028705 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
@@ -30,7 +30,8 @@ import eu.stratosphere.configuration.Configuration;
 /**
  * The <tt>TwoInputUdfOperator</tt> is the base class of all binary operators that execute
  * user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have two inputs (such as {@link JoinFunction} or {@link CoGroupFunction}).
+ * have two inputs (such as {@link eu.stratosphere.api.java.functions.JoinFunction} or 
+ * {@link eu.stratosphere.api.java.functions.CoGroupFunction}).
  * <p>
  * This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
  * through configuration objects, and semantic properties.
@@ -103,7 +104,8 @@ public abstract class TwoInputUdfOperator<IN1, IN2, OUT, O extends TwoInputUdfOp
 	 * <p>
 	 * Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
 	 * In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
-	 * Constant set annotations can only be specified if the first input and the output type of the UDF are of {@link Tuple} data types.
+	 * Constant set annotations can only be specified if the first input and the output type of the UDF are of
+	 * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
 	 * 
 	 * <p>
 	 * A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of 
@@ -134,7 +136,8 @@ public abstract class TwoInputUdfOperator<IN1, IN2, OUT, O extends TwoInputUdfOp
 	 * <p>
 	 * Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
 	 * In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
-	 * Constant set annotations can only be specified if the second input and the output type of the UDF are of {@link Tuple} data types.
+	 * Constant set annotations can only be specified if the second input and the output type of the UDF are of
+	 * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
 	 * 
 	 * <p>
 	 * A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of