You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/07/03 14:51:02 UTC
[1/3] git commit: [FLINK-996] Fix for bug in union replacement
Repository: incubator-flink
Updated Branches:
refs/heads/master 8a8060eaa -> 32a003d5c
[FLINK-996] Fix for bug in union replacement
This closes #56
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/32a003d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/32a003d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/32a003d5
Branch: refs/heads/master
Commit: 32a003d5c0f507f4634851434e8ffb8ce6b880b2
Parents: cf29798
Author: Till Rohrmann <ti...@gmail.com>
Authored: Wed Jul 2 16:00:55 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200
----------------------------------------------------------------------
.../eu/stratosphere/compiler/PactCompiler.java | 53 ++++++--------------
.../pact/compiler/UnionReplacementTest.java | 49 ++++++++++++++++++
2 files changed, 64 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/32a003d5/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
----------------------------------------------------------------------
diff --git a/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java b/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
index bf3d6af..5a0a3e1 100644
--- a/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
+++ b/stratosphere-compiler/src/main/java/eu/stratosphere/compiler/PactCompiler.java
@@ -1195,7 +1195,7 @@ public class PactCompiler {
private static final class BinaryUnionReplacer implements Visitor<PlanNode> {
private final Set<PlanNode> seenBefore = new HashSet<PlanNode>();
-
+
@Override
public boolean preVisit(PlanNode visitable) {
if (this.seenBefore.add(visitable)) {
@@ -1217,43 +1217,20 @@ public class PactCompiler {
final Channel in2 = unionNode.getInput2();
PlanNode newUnionNode;
-
- // if any input is cached, we keep this as a binary union and do not collapse it into a
- // n-ary union
-// if (in1.getTempMode().isCached() || in2.getTempMode().isCached()) {
-// // replace this node by an explicit operator
-// Channel cached, pipelined;
-// if (in1.getTempMode().isCached()) {
-// cached = in1;
-// pipelined = in2;
-// } else {
-// cached = in2;
-// pipelined = in1;
-// }
-//
-// newUnionNode = new DualInputPlanNode(unionNode.getOriginalOptimizerNode(), cached, pipelined,
-// DriverStrategy.UNION_WITH_CACHED);
-// newUnionNode.initProperties(unionNode.getGlobalProperties(), new LocalProperties());
-//
-// in1.setTarget(newUnionNode);
-// in2.setTarget(newUnionNode);
-// } else {
- // collect the union inputs to collapse this operator with
- // its collapsed predecessors. check whether an input is materialized to prevent
- // collapsing
- List<Channel> inputs = new ArrayList<Channel>();
- collect(in1, inputs);
- collect(in2, inputs);
-
- newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties());
-
- // adjust the input channels to have their target point to the new union node
- for (Channel c : inputs) {
- c.setTarget(newUnionNode);
- }
-// }
-
- unionNode.getOutgoingChannels().get(0).swapUnionNodes(newUnionNode);
+
+ List<Channel> inputs = new ArrayList<Channel>();
+ collect(in1, inputs);
+ collect(in2, inputs);
+
+ newUnionNode = new NAryUnionPlanNode(unionNode.getOptimizerNode(), inputs, unionNode.getGlobalProperties());
+
+ for (Channel c : inputs) {
+ c.setTarget(newUnionNode);
+ }
+
+ for(Channel channel : unionNode.getOutgoingChannels()){
+ channel.swapUnionNodes(newUnionNode);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/32a003d5/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
----------------------------------------------------------------------
diff --git a/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java b/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
new file mode 100644
index 0000000..9ef8519
--- /dev/null
+++ b/stratosphere-compiler/src/test/java/eu/stratosphere/pact/compiler/UnionReplacementTest.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package eu.stratosphere.pact.compiler;
+
+import eu.stratosphere.api.common.Plan;
+import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.ExecutionEnvironment;
+import eu.stratosphere.api.java.IterativeDataSet;
+import eu.stratosphere.compiler.CompilerException;
+import eu.stratosphere.compiler.plan.OptimizedPlan;
+import eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator;
+import org.junit.Test;
+import static org.junit.Assert.fail;
+
+public class UnionReplacementTest extends CompilerTestBase {
+
+ @Test
+ public void testUnionReplacement(){
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<String> input1 = env.fromElements("test1");
+ DataSet<String> input2 = env.fromElements("test2");
+
+ DataSet<String> union = input1.union(input2);
+
+ union.print();
+ union.print();
+
+ Plan plan = env.createProgramPlan();
+ try{
+ OptimizedPlan oPlan = this.compileNoStats(plan);
+ NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
+ jobGen.compileJobGraph(oPlan);
+ }catch(CompilerException co){
+ co.printStackTrace();
+ fail("The Pact compiler is unable to compile this plan correctly.");
+ }
+ }
+}
[3/3] git commit: [FLINK-932] Introduce automatic generation of
semantic props for projections
Posted by se...@apache.org.
[FLINK-932] Introduce automatic generation of semantic props for projections
This closes #29
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/883474dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/883474dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/883474dd
Branch: refs/heads/master
Commit: 883474ddfd749eedceb85bdd2925a3a76de75206
Parents: 8a8060e
Author: sebastian kunert <sk...@gmail.com>
Authored: Thu Jun 19 16:17:59 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200
----------------------------------------------------------------------
.../api/java/functions/SemanticPropUtil.java | 21 +++
.../api/java/operators/CrossOperator.java | 34 ++++-
.../api/java/operators/JoinOperator.java | 36 ++++-
.../api/java/operators/ProjectOperator.java | 5 +-
.../SemanticPropertiesProjectionTest.java | 144 +++++++++++++++++++
5 files changed, 233 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
index 4d767b0..9f661f0 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/functions/SemanticPropUtil.java
@@ -48,6 +48,27 @@ public class SemanticPropUtil {
private static final Pattern PATTERN_DIGIT = Pattern.compile("\\d+");
+ public static SingleInputSemanticProperties createProjectionPropertiesSingle(int[] fields) {
+ SingleInputSemanticProperties ssp = new SingleInputSemanticProperties();
+ for (int i = 0; i < fields.length; i++) {
+ ssp.addForwardedField(fields[i], i);
+ }
+ return ssp;
+ }
+
+ public static DualInputSemanticProperties createProjectionPropertiesDual(int[] fields, boolean[] isFromFirst) {
+ DualInputSemanticProperties dsp = new DualInputSemanticProperties();
+
+ for (int i = 0; i < fields.length; i++) {
+ if (isFromFirst[i]) {
+ dsp.addForwardedField1(fields[i], i);
+ } else {
+ dsp.addForwardedField2(fields[i], i);
+ }
+ }
+ return dsp;
+ }
+
public static SingleInputSemanticProperties getSemanticPropsSingle(Set<Annotation> set, TypeInformation<?> inType, TypeInformation<?> outType) {
if (set == null) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
index b497407..76e5320 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
@@ -18,10 +18,12 @@ import java.util.Arrays;
import eu.stratosphere.api.common.functions.GenericCrosser;
import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.api.common.operators.base.CrossOperatorBase;
import eu.stratosphere.api.java.DataSet;
import eu.stratosphere.api.java.functions.CrossFunction;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.typeutils.TupleTypeInfo;
import eu.stratosphere.api.java.typeutils.TypeExtractor;
//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
@@ -49,9 +51,19 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
super(input1, input2, returnType);
this.function = function;
- extractSemanticAnnotationsFromUdf(function.getClass());
+
+ if (!(function instanceof ProjectCrossFunction)) {
+ extractSemanticAnnotationsFromUdf(function.getClass());
+ } else {
+ generateProjectionProperties(((ProjectCrossFunction) function));
+ }
+ }
+
+ public void generateProjectionProperties(ProjectCrossFunction pcf) {
+ DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pcf.getFields(), pcf.getIsFromFirst());
+ setSemanticProperties(props);
}
-
+
@Override
protected eu.stratosphere.api.common.operators.base.CrossOperatorBase<I1, I2, OUT, GenericCrosser<I1,I2,OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
@@ -187,6 +199,16 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
super(input1, input2,
new ProjectCrossFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()), returnType);
}
+
+ @Override
+ public CrossOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
+ throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+ }
+
+ @Override
+ public CrossOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
+ throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+ }
}
public static final class ProjectCrossFunction<T1, T2, R extends Tuple> extends CrossFunction<T1, T2, R> {
@@ -234,6 +256,14 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
}
return outTuple;
}
+
+ protected int[] getFields() {
+ return fields;
+ }
+
+ protected boolean[] getIsFromFirst() {
+ return isFromFirst;
+ }
}
public static final class CrossProjection<I1, I2> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
index 9479c28..c96f05c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
@@ -21,6 +21,7 @@ import eu.stratosphere.api.common.InvalidProgramException;
import eu.stratosphere.api.common.functions.GenericJoiner;
import eu.stratosphere.api.common.functions.GenericMap;
import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.api.common.operators.UnaryOperatorInformation;
import eu.stratosphere.api.common.operators.base.JoinOperatorBase;
@@ -29,6 +30,7 @@ import eu.stratosphere.api.java.DataSet;
import eu.stratosphere.api.java.DeltaIteration.SolutionSetPlaceHolder;
import eu.stratosphere.api.java.functions.JoinFunction;
import eu.stratosphere.api.java.functions.KeySelector;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.operators.Keys.FieldPositionKeys;
import eu.stratosphere.api.java.operators.translation.KeyExtractingMapper;
import eu.stratosphere.api.java.operators.translation.PlanUnwrappingJoinOperator;
@@ -163,9 +165,19 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
}
this.function = function;
- extractSemanticAnnotationsFromUdf(function.getClass());
+
+ if (!(function instanceof ProjectJoinFunction)) {
+ extractSemanticAnnotationsFromUdf(function.getClass());
+ } else {
+ generateProjectionProperties(((ProjectJoinFunction) function));
+ }
}
-
+
+ public void generateProjectionProperties(ProjectJoinFunction pjf) {
+ DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pjf.getFields(), pjf.getIsFromFirst());
+ setSemanticProperties(props);
+ }
+
// TODO
// public EquiJoin<I1, I2, OUT> leftOuter() {
// this.preserve1 = true;
@@ -512,6 +524,16 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
new ProjectJoinFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()),
returnType, hint);
}
+
+ @Override
+ public JoinOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
+ throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+ }
+
+ @Override
+ public JoinOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
+ throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+ }
}
// @SuppressWarnings("unused")
@@ -824,7 +846,15 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
this.isFromFirst = isFromFirst;
this.outTuple = outTupleInstance;
}
-
+
+ protected int[] getFields() {
+ return fields;
+ }
+
+ protected boolean[] getIsFromFirst() {
+ return isFromFirst;
+ }
+
public R join(T1 in1, T2 in2) {
for(int i=0; i<fields.length; i++) {
if(isFromFirst[i]) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
index 1907393..35406d6 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/ProjectOperator.java
@@ -19,6 +19,7 @@ import java.util.Arrays;
import eu.stratosphere.api.common.functions.GenericMap;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.functions.SemanticPropUtil;
import eu.stratosphere.api.java.operators.translation.PlanProjectOperator;
//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
import eu.stratosphere.api.java.tuple.*;
@@ -54,11 +55,11 @@ public class ProjectOperator<IN, OUT extends Tuple>
ppo.setInput(input);
// set dop
ppo.setDegreeOfParallelism(this.getParallelism());
-
+ ppo.setSemanticProperties(SemanticPropUtil.createProjectionPropertiesSingle(fields));
+
return ppo;
}
-
public static class Projection<T> {
private final DataSet<T> ds;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/883474dd/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java b/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
new file mode 100644
index 0000000..4e812da
--- /dev/null
+++ b/stratosphere-java/src/test/java/eu/stratosphere/api/java/functions/SemanticPropertiesProjectionTest.java
@@ -0,0 +1,144 @@
+/***********************************************************************************************************************
+ *
+ * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ **********************************************************************************************************************/
+
+package eu.stratosphere.api.java.functions;
+
+import eu.stratosphere.api.common.Plan;
+import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
+import eu.stratosphere.api.common.operators.SingleInputSemanticProperties;
+import eu.stratosphere.api.common.operators.base.CrossOperatorBase;
+import eu.stratosphere.api.common.operators.base.GenericDataSinkBase;
+import eu.stratosphere.api.common.operators.base.JoinOperatorBase;
+import eu.stratosphere.api.java.DataSet;
+import eu.stratosphere.api.java.ExecutionEnvironment;
+import eu.stratosphere.api.java.operators.translation.PlanProjectOperator;
+import eu.stratosphere.api.java.tuple.Tuple4;
+import eu.stratosphere.api.java.tuple.Tuple5;
+import eu.stratosphere.api.java.typeutils.BasicTypeInfo;
+import eu.stratosphere.api.java.typeutils.TupleTypeInfo;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Created by sebastian on 6/19/14.
+ */
+public class SemanticPropertiesProjectionTest {
+
+ final List<Tuple5<Integer, Long, String, Long, Integer>> emptyTupleData = new ArrayList<Tuple5<Integer, Long, String, Long, Integer>>();
+
+ final TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>> tupleTypeInfo = new
+ TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO
+ );
+
+
+ @Test
+ public void ProjectOperatorTest() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+ tupleDs.project(1, 3, 2).types(Long.class, Long.class, String.class).print();
+
+ Plan plan = env.createProgramPlan();
+
+ GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+ PlanProjectOperator<?, ?> projectOperator = ((PlanProjectOperator) sink.getInput());
+
+ SingleInputSemanticProperties props = projectOperator.getSemanticProperties();
+
+ assertTrue(props.getForwardedField(1).size() == 1);
+ assertTrue(props.getForwardedField(3).size() == 1);
+ assertTrue(props.getForwardedField(2).size() == 1);
+ assertTrue(props.getForwardedField(1).contains(0));
+ assertTrue(props.getForwardedField(3).contains(1));
+ assertTrue(props.getForwardedField(2).contains(2));
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Exception in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void JoinProjectionTest() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+ tupleDs.join(tupleDs).where(0).equalTo(0).projectFirst(2, 3).projectSecond(1, 4).types(String.class, Long.class, Long.class, Integer.class).print();
+
+ Plan plan = env.createProgramPlan();
+
+ GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+ JoinOperatorBase<?, ?, ?, ?> projectJoinOperator = ((JoinOperatorBase<?, ?, ?, ?>) sink.getInput());
+
+ DualInputSemanticProperties props = projectJoinOperator.getSemanticProperties();
+
+ assertTrue(props.getForwardedField1(2).size() == 1);
+ assertTrue(props.getForwardedField1(3).size() == 1);
+ assertTrue(props.getForwardedField2(1).size() == 1);
+ assertTrue(props.getForwardedField2(4).size() == 1);
+ assertTrue(props.getForwardedField1(2).contains(0));
+ assertTrue(props.getForwardedField1(3).contains(1));
+ assertTrue(props.getForwardedField2(1).contains(2));
+ assertTrue(props.getForwardedField2(4).contains(3));
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Exception in test: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void CrossProjectionTest() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
+
+ DataSet<Tuple4<String, Long, Long, Integer>> result = tupleDs.cross(tupleDs).projectFirst(2, 3).projectSecond(1, 4).types(String.class, Long.class, Long.class, Integer.class);
+ result.print();
+
+ Plan plan = env.createProgramPlan();
+
+ GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
+ CrossOperatorBase<?, ?, ?, ?> projectCrossOperator = ((CrossOperatorBase<?, ?, ?, ?>) sink.getInput());
+
+ DualInputSemanticProperties props = projectCrossOperator.getSemanticProperties();
+
+ assertTrue(props.getForwardedField1(2).size() == 1);
+ assertTrue(props.getForwardedField1(3).size() == 1);
+ assertTrue(props.getForwardedField2(1).size() == 1);
+ assertTrue(props.getForwardedField2(4).size() == 1);
+ assertTrue(props.getForwardedField1(2).contains(0));
+ assertTrue(props.getForwardedField1(3).contains(1));
+ assertTrue(props.getForwardedField2(1).contains(2));
+ assertTrue(props.getForwardedField2(4).contains(3));
+ } catch (Exception e) {
+ System.err.println(e.getMessage());
+ e.printStackTrace();
+ fail("Exception in test: " + e.getMessage());
+ }
+ }
+}
[2/3] git commit: Fix various error messages and invalid JavaDocs.
Posted by se...@apache.org.
Fix various error messages and invalid JavaDocs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/cf29798b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/cf29798b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/cf29798b
Branch: refs/heads/master
Commit: cf29798b20572db29051c0c67658e18ac92e998e
Parents: 883474d
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 3 13:33:53 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 3 14:49:39 2014 +0200
----------------------------------------------------------------------
.../api/java/operators/CrossOperator.java | 52 ++++++++------
.../api/java/operators/JoinOperator.java | 73 +++++++++++---------
.../java/operators/SingleInputUdfOperator.java | 6 +-
.../api/java/operators/TwoInputUdfOperator.java | 9 ++-
4 files changed, 79 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
index 76e5320..1b0bc0c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/CrossOperator.java
@@ -16,6 +16,7 @@ package eu.stratosphere.api.java.operators;
import java.util.Arrays;
+import eu.stratosphere.api.common.InvalidProgramException;
import eu.stratosphere.api.common.functions.GenericCrosser;
import eu.stratosphere.api.common.operators.BinaryOperatorInformation;
import eu.stratosphere.api.common.operators.DualInputSemanticProperties;
@@ -55,11 +56,11 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
if (!(function instanceof ProjectCrossFunction)) {
extractSemanticAnnotationsFromUdf(function.getClass());
} else {
- generateProjectionProperties(((ProjectCrossFunction) function));
+ generateProjectionProperties(((ProjectCrossFunction<?, ?, ?>) function));
}
}
- public void generateProjectionProperties(ProjectCrossFunction pcf) {
+ public void generateProjectionProperties(ProjectCrossFunction<?, ?, ?> pcf) {
DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pcf.getFields(), pcf.getIsFromFirst());
setSemanticProperties(props);
}
@@ -92,7 +93,6 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
*
* @param <I1> The type of the first input DataSet of the Cross transformation.
* @param <I2> The type of the second input DataSet of the Cross transformation.
- * @param <OUT> The type of the result of the Cross transformation.
*
* @see Tuple2
* @see DataSet
@@ -140,18 +140,20 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
* If the first cross input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
+ * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
- * @return A CrossProjection that needs to be converted into a {@link ProjectCross} to complete the
+ * @return A CrossProjection that needs to be converted into a
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.ProjectCross} to complete the
* Cross transformation by calling {@link CrossProjection#types()}.
*
* @see Tuple
* @see DataSet
- * @see CrossProjection
- * @see ProjectCross
+ * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+ * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
*/
public CrossProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
return new CrossProjection<I1, I2>(input1, input2, firstFieldIndexes, null);
@@ -163,18 +165,20 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
* If the second cross input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+ * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
- * @return A CrossProjection that needs to be converted into a {@link ProjectCross} to complete the
+ * @return A CrossProjection that needs to be converted into a
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.ProjectCross} to complete the
* Cross transformation by calling {@link CrossProjection#types()}.
*
* @see Tuple
* @see DataSet
- * @see CrossProjection
- * @see ProjectCross
+ * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+ * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
*/
public CrossProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
return new CrossProjection<I1, I2>(input1, input2, null, secondFieldIndexes);
@@ -202,12 +206,12 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
@Override
public CrossOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
- throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+ throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
}
@Override
public CrossOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
- throw new RuntimeException("Please do not use ConstantFields on ProjectCross. The Fields are automatically calculated.");
+ throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
}
}
@@ -364,9 +368,10 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
* If the first cross input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
+ * @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
* @return A CrossProjection that needs to be converted into a {@link ProjectOperator} to complete the
@@ -374,8 +379,8 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
*
* @see Tuple
* @see DataSet
- * @see CrossProjection
- * @see ProjectCross
+ * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+ * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
*/
public CrossProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
@@ -431,9 +436,10 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
* If the second cross input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link CrossProjection#projectFirst(int...)} and {@link CrossProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.CrossOperator.CrossProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+ * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
* @return A CrossProjection that needs to be converted into a {@link ProjectOperator} to complete the
@@ -441,8 +447,8 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
*
* @see Tuple
* @see DataSet
- * @see CrossProjection
- * @see ProjectCross
+ * @see eu.stratosphere.api.java.operators.CrossOperator.CrossProjection
+ * @see eu.stratosphere.api.java.operators.CrossOperator.ProjectCross
*/
public CrossProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
index c96f05c..509585c 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/JoinOperator.java
@@ -169,11 +169,11 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
if (!(function instanceof ProjectJoinFunction)) {
extractSemanticAnnotationsFromUdf(function.getClass());
} else {
- generateProjectionProperties(((ProjectJoinFunction) function));
+ generateProjectionProperties(((ProjectJoinFunction<?, ?, ?>) function));
}
}
- public void generateProjectionProperties(ProjectJoinFunction pjf) {
+ public void generateProjectionProperties(ProjectJoinFunction<?, ?, ?> pjf) {
DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pjf.getFields(), pjf.getIsFromFirst());
setSemanticProperties(props);
}
@@ -431,7 +431,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @return An EquiJoin that represents the joined result DataSet
*
* @see JoinFunction
- * @see EquiJoin
+ * @see eu.stratosphere.api.java.operators.JoinOperator.EquiJoin
* @see DataSet
*/
public <R> EquiJoin<I1, I2, R> with(JoinFunction<I1, I2, R> function) {
@@ -448,18 +448,20 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* If the first join input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
*
* @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
- * @return A JoinProjection that needs to be converted into a {@link ProjectJoin} to complete the
- * Join transformation by calling {@link JoinProjection#types()}.
+ * @return A JoinProjection that needs to be converted into a
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin} to complete the
+ * Join transformation by calling the corresponding {@code types()} function.
*
* @see Tuple
* @see DataSet
- * @see JoinProjection
- * @see ProjectJoin
+ * @see eu.stratosphere.api.java.operators.JoinOperator.JoinProjection
+ * @see eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin
*/
public JoinProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
return new JoinProjection<I1, I2>(getInput1(), getInput2(), getKeys1(), getKeys2(), getJoinHint(), firstFieldIndexes, null);
@@ -471,18 +473,20 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* If the second join input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+ * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
- * @return A JoinProjection that needs to be converted into a {@link ProjectJoin} to complete the
- * Join transformation by calling {@link JoinProjection#types()}.
+ * @return A JoinProjection that needs to be converted into a
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin} to complete the
+ * Join transformation by calling the corresponding {@code types()} function.
*
* @see Tuple
* @see DataSet
- * @see JoinProjection
- * @see ProjectJoin
+ * @see eu.stratosphere.api.java.operators.JoinOperator.JoinProjection
+ * @see eu.stratosphere.api.java.operators.JoinOperator.ProjectJoin
*/
public JoinProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
return new JoinProjection<I1, I2>(getInput1(), getInput2(), getKeys1(), getKeys2(), getJoinHint(), null, secondFieldIndexes);
@@ -527,12 +531,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
@Override
public JoinOperator<I1, I2, OUT> withConstantSetFirst(String... constantSetFirst) {
- throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+ throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
}
@Override
public JoinOperator<I1, I2, OUT> withConstantSetSecond(String... constantSetSecond) {
- throw new RuntimeException("Please do not use ConstantFields on ProjectJoins. The Fields are automatically calculated.");
+ throw new InvalidProgramException("The semantic properties (constant fields and forwarded fields) are automatically calculated.");
}
}
@@ -601,7 +605,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
/**
* Intermediate step of a Join transformation. <br/>
* To continue the Join transformation, select the join key of the first input {@link DataSet} by calling
- * {@link JoinOperatorSets#where(int...)} or {@link JoinOperatorSets#where(KeySelector)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets#where(int, int...)} or
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets#where(KeySelector)}.
*
* @param <I1> The type of the first input DataSet of the Join transformation.
* @param <I2> The type of the second input DataSet of the Join transformation.
@@ -635,7 +640,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param field0 The first index of the Tuple fields of the first join DataSets that should be used as key
* @param fields The indexes of the other Tuple fields of the first join DataSets that should be used as keys.
* @return An incomplete Join transformation.
- * Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+ * Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
* to continue the Join.
*
* @see Tuple
@@ -656,7 +662,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param field0 The first field of the Tuple fields of the first join DataSets that should be used as key
* @param fields The fields of the first join DataSets that should be used as keys.
* @return An incomplete Join transformation.
- * Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+ * Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
* to continue the Join.
*
* @see Tuple
@@ -676,7 +683,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
*
* @param keySelector The KeySelector function which extracts the key values from the DataSet on which it is joined.
* @return An incomplete Join transformation.
- * Call {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}
+ * Call {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}
* to continue the Join.
*
* @see KeySelector
@@ -691,7 +699,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
/**
* Intermediate step of a Join transformation. <br/>
* To continue the Join transformation, select the join key of the second input {@link DataSet} by calling
- * {@link JoinOperatorSetsPredicate#equalTo(int...)} or {@link JoinOperatorSetsPredicate#equalTo(KeySelector)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(int, int...)} or
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinOperatorSets.JoinOperatorSetsPredicate#equalTo(KeySelector)}.
*
*/
public class JoinOperatorSetsPredicate {
@@ -715,7 +724,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* {@link DataSet} that should be used as join keys.<br/>
* <b>Note: Fields can only be selected as join keys on Tuple DataSets.</b><br/>
*
- * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
+ * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
* the element of the first input being the first field of the tuple and the element of the
* second input being the second field of the tuple.
*
@@ -734,7 +743,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* Continues a Join transformation and defines the fields of the second join
* {@link DataSet} that should be used as join keys.<br/>
*
- * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
+ * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
* the element of the first input being the first field of the tuple and the element of the
* second input being the second field of the tuple.
*
@@ -754,7 +763,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* The KeySelector function is called for each element of the second DataSet and extracts a single
* key value on which the DataSet is joined. </br>
*
- * The resulting {@link DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
+ * The resulting {@link eu.stratosphere.api.java.operators.JoinOperator.DefaultJoin} wraps each pair of joining elements into a {@link Tuple2}, with
* the element of the first input being the first field of the tuple and the element of the
* second input being the second field of the tuple.
*
@@ -999,18 +1008,17 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* If the first join input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
*
* @param firstFieldIndexes If the first input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
* @return A JoinProjection that needs to be converted into a {@link ProjectOperator} to complete the
- * ProjectJoin transformation by calling {@link JoinProjection#types()}.
+ * ProjectJoin transformation by calling the corresponding {@code types()} function.
*
* @see Tuple
* @see DataSet
- * @see JoinProjection
- * @see ProjectJoin
*/
public JoinProjection<I1, I2> projectFirst(int... firstFieldIndexes) {
@@ -1066,18 +1074,17 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* If the second join input is not a Tuple DataSet, no parameters should be passed.<br/>
*
* Fields of the first and second input can be added by chaining the method calls of
- * {@link JoinProjection#projectFirst(int...)} and {@link JoinProjection#projectSecond(int...)}.
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectFirst(int...)} and
+ * {@link eu.stratosphere.api.java.operators.JoinOperator.JoinProjection#projectSecond(int...)}.
*
- * @param fieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
+ * @param secondFieldIndexes If the second input is a Tuple DataSet, the indexes of the selected fields.
* For a non-Tuple DataSet, do not provide parameters.
* The order of fields in the output tuple is defined by to the order of field indexes.
* @return A JoinProjection that needs to be converted into a {@link ProjectOperator} to complete the
- * ProjectJoin transformation by calling {@link JoinProjection#types()}.
+ * ProjectJoin transformation by calling the corresponding {@code types()} function.
*
* @see Tuple
* @see DataSet
- * @see JoinProjection
- * @see ProjectJoin
*/
public JoinProjection<I1, I2> projectSecond(int... secondFieldIndexes) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
index 75e8cb5..4e9c567 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/SingleInputUdfOperator.java
@@ -30,7 +30,8 @@ import eu.stratosphere.configuration.Configuration;
/**
* The <tt>SingleInputUdfOperator</tt> is the base class of all unary operators that execute
* user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have one input (such as {@link MapFunction} or {@link ReduceFunction}).
+ * have one input (such as {@link eu.stratosphere.api.java.functions.MapFunction} or
+ * {@link eu.stratosphere.api.java.functions.ReduceFunction}).
* <p>
* This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
* through configuration objects, and semantic properties.
@@ -98,7 +99,8 @@ public abstract class SingleInputUdfOperator<IN, OUT, O extends SingleInputUdfOp
* <p>
* Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
* In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
- * Constant set annotations can only be specified if the second input and the output type of the UDF are of {@link Tuple} data types.
+ * Constant set annotations can only be specified if the second input and the output type of the UDF are of
+ * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
*
* <p>
* A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cf29798b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
index ef622de..c028705 100644
--- a/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
+++ b/stratosphere-java/src/main/java/eu/stratosphere/api/java/operators/TwoInputUdfOperator.java
@@ -30,7 +30,8 @@ import eu.stratosphere.configuration.Configuration;
/**
* The <tt>TwoInputUdfOperator</tt> is the base class of all binary operators that execute
* user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have two inputs (such as {@link JoinFunction} or {@link CoGroupFunction}).
+ * have two inputs (such as {@link eu.stratosphere.api.java.functions.JoinFunction} or
+ * {@link eu.stratosphere.api.java.functions.CoGroupFunction}).
* <p>
* This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
* through configuration objects, and semantic properties.
@@ -103,7 +104,8 @@ public abstract class TwoInputUdfOperator<IN1, IN2, OUT, O extends TwoInputUdfOp
* <p>
* Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
* In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
- * Constant set annotations can only be specified if the first input and the output type of the UDF are of {@link Tuple} data types.
+ * Constant set annotations can only be specified if the first input and the output type of the UDF are of
+ * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
*
* <p>
* A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of
@@ -134,7 +136,8 @@ public abstract class TwoInputUdfOperator<IN1, IN2, OUT, O extends TwoInputUdfOp
* <p>
* Constant set annotations are used by the optimizer to infer the existence of data properties (sorted, partitioned, grouped).
* In certain cases, these annotations allow the optimizer to generate a more efficient execution plan which can lead to improved performance.
- * Constant set annotations can only be specified if the second input and the output type of the UDF are of {@link Tuple} data types.
+ * Constant set annotations can only be specified if the second input and the output type of the UDF are of
+ * {@link eu.stratosphere.api.java.tuple.Tuple} data types.
*
* <p>
* A constant-set annotation is a set of constant field specifications. The constant field specification String "4->3" specifies, that this UDF copies the fourth field of