You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2014/08/09 14:39:32 UTC
[01/39] git commit: [FLINK-1024] Enable automated snapshot deployment
through Apache's Maven infrastructure
Repository: incubator-flink
Updated Branches:
refs/heads/travis_test de13d1737 -> d83c81086 (forced update)
[FLINK-1024] Enable automated snapshot deployment through Apache's Maven infrastructure
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/dc55cc35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/dc55cc35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/dc55cc35
Branch: refs/heads/travis_test
Commit: dc55cc351b4f95f05411b47639f33155194613d3
Parents: 3827f06
Author: Robert Metzger <rm...@apache.org>
Authored: Tue Jul 22 12:10:07 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Tue Jul 22 16:59:19 2014 +0200
----------------------------------------------------------------------
.travis.yml | 6 +-
deploysettings.xml | 4 +-
.../main/resources/archetype-resources/pom.xml | 14 ++
.../main/resources/archetype-resources/pom.xml | 15 +-
flink-quickstart/pom.xml | 142 ++++++++-----------
flink-quickstart/quickstart-SNAPSHOT.sh | 2 +-
flink-quickstart/quickstart-scala-SNAPSHOT.sh | 2 +-
tools/deploy_to_maven.sh | 44 +++---
8 files changed, 120 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index d36473a..c3899a7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,9 +15,9 @@ notifications:
env:
global:
- # username and password for sonatype (maven deploy)
- - secure: "iQrrSH9vL/ANeqj0u2qC1Ft4eAor2KF6mrqA4T6/5Woxqoq22siw9SGcSW7UmLr3fpbzqsapK8XX8Rkpgsi6V7YTpACZyj0tIX6knoy08KK4qJaGEDx+8FlM61l9qQCQ5gFVVu+cFvI+xd3tDCoZuGSZUjZOjNpYZZqSunsOZ7E="
- - secure: "aip84Xhcy9jYUtLrdfL4uWNATmLgqwIU/Gn9dR85uPJHeVPq7gSq3yrmiLJkEUgc6WyLdsKmiGKPPqwAG2Hc57i/cdz3YSq0Cn/Ksuxviec66HvNIhIFLwuXQkWF/W7MLmMgL6kVD8PxMn+A3d/4aEUKX4ELhoAmBhULOYiXi+E="
+ # username and password for Apache Nexus (maven deploy)
+ - secure: "Nu2oNTrIAmxIkNEZzALw+GT2QBogEh/mqecSqoKDCk0oFjUZhrnrsIZYD/8zTG9fAVa5Gx4uWH4W824Va5RlBZvCs9UTh5TF25K2ORR9dB9FiXZ+Vjjig78sKJF7N73WVIOsHCSKpoBKnVkvNwxuAkPTMYjn3sswRh1pMu2VQ90="
+ - secure: "jMllQXAHpE+ijYXjvQvh0xml6DCL5pmESuWRtd0Wi4v56HHxKHc/Tty/CJvX8whVDLaHNFtwlbaIN9asSyAu1OyGhpWCqsmsxWF4atvKFua1oX45XMB26Ymf7Yr7aq7lcx66j0cYpfBXY4tFTFPiT05QnZ8XsHzEnv4Tpgif2dg="
# New s3 deployment
- ARTIFACTS_S3_BUCKET="stratosphere-bin"
- secure: "AECzVxihEhYfnNcrY/wLirTkKkmSATycvTfKsBmxD07bg6BmaVgsOl4degUu4YL50e6agpoWul6irGxTg0bjLMAwg1ZGyRx57NFvNQ7JYDHK6EWmJ7BsK2WO7HiYzfau+ZAaL36WpOMi0UUPpuNXMvULqaE9b4jZqo1Wo/WDcyU="
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/deploysettings.xml
----------------------------------------------------------------------
diff --git a/deploysettings.xml b/deploysettings.xml
index be19de1..e36d848 100644
--- a/deploysettings.xml
+++ b/deploysettings.xml
@@ -24,12 +24,12 @@ under the License.
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
- <id>sonatype-nexus-snapshots</id>
+ <id>apache.snapshots.https</id>
<username>${sonatype_user}</username>
<password>${sonatype_pw}</password>
</server>
<server>
- <id>sonatype-nexus-staging</id>
+ <id>apache.releases.https</id>
<username>${sonatype_user}</username>
<password>${sonatype_pw}</password>
</server>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index c7bb513..140792d 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -32,6 +32,20 @@ under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Development Snapshot Repository</name>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
<!-- These two requirements are the minimum to use and develop Flink.
You can add others like <artifactId>pact-scala-core</artifactId> for Scala! -->
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 14922bb..0580447 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,20 @@ under the License.
<name>Your Job's Name</name>
<url>http://www.myorganization.org</url>
-
+
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <name>Apache Development Snapshot Repository</name>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/flink-quickstart/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml
index 665555f..9860295 100644
--- a/flink-quickstart/pom.xml
+++ b/flink-quickstart/pom.xml
@@ -33,91 +33,75 @@ under the License.
<name>flink-quickstart</name>
<description>Parent project for different quickstart archetypes for Apache Flink</description>
+ <modules>
+ <module>flink-quickstart-java</module>
+ <module>flink-quickstart-scala</module>
+ </modules>
- <!-- See http://www.imixs.org/jee/archetype/build.html -->
- <!-- Distributen Management oss.sonatype.org -->
- <distributionManagement>
- <snapshotRepository>
- <id>sonatype-nexus-snapshots</id>
- <name>Sonatype Nexus Snapshots</name>
- <url>http://oss.sonatype.org/content/repositories/snapshots</url>
- </snapshotRepository>
- <repository>
- <id>sonatype-nexus-staging</id>
- <name>Nexus Release Repository</name>
- <url>http://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
- </repository>
- </distributionManagement>
-
- <modules>
- <module>flink-quickstart-java</module>
- <module>flink-quickstart-scala</module>
- </modules>
-
- <profiles>
- <profile>
- <id>release</id>
- <build>
- <plugins>
- <!-- source attachment -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <version>2.2.1</version>
- <executions>
- <execution>
- <id>attach-sources</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <!-- Javadocs -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.9.1</version>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <!-- signing -->
- <plugin>
+ <profiles>
+ <profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <!-- source attachment -->
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-gpg-plugin</artifactId>
- <version>1.4</version>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
<executions>
<execution>
- <id>sign-artifacts</id>
- <phase>verify</phase>
- <goals>
- <goal>sign</goal>
- </goals>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
</execution>
</executions>
+ </plugin>
+ <!-- Javadocs -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- signing -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <version>1.4</version>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <version>2.1</version>
+ <configuration>
+ <mavenExecutorId>forked-path</mavenExecutorId>
+ <useReleaseProfile>false</useReleaseProfile>
+ <arguments>${arguments} -Psonatype-oss-release</arguments>
+ </configuration>
</plugin>
</plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-release-plugin</artifactId>
- <version>2.1</version>
- <configuration>
- <mavenExecutorId>forked-path</mavenExecutorId>
- <useReleaseProfile>false</useReleaseProfile>
- <arguments>${arguments} -Psonatype-oss-release</arguments>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
- </profile>
- </profiles>
+ </pluginManagement>
+ </build>
+ </profile>
+ </profiles>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/flink-quickstart/quickstart-SNAPSHOT.sh
----------------------------------------------------------------------
diff --git a/flink-quickstart/quickstart-SNAPSHOT.sh b/flink-quickstart/quickstart-SNAPSHOT.sh
index be0b461..b54c117 100755
--- a/flink-quickstart/quickstart-SNAPSHOT.sh
+++ b/flink-quickstart/quickstart-SNAPSHOT.sh
@@ -30,7 +30,7 @@ mvn archetype:generate \
-Dversion=0.1 \
-Dpackage=org.apache.flink \
-DinteractiveMode=false \
- -DarchetypeCatalog=https://oss.sonatype.org/content/repositories/snapshots/
+ -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/
#
# Give some guidance
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/flink-quickstart/quickstart-scala-SNAPSHOT.sh
----------------------------------------------------------------------
diff --git a/flink-quickstart/quickstart-scala-SNAPSHOT.sh b/flink-quickstart/quickstart-scala-SNAPSHOT.sh
index e8e1c58..f9701ac 100755
--- a/flink-quickstart/quickstart-scala-SNAPSHOT.sh
+++ b/flink-quickstart/quickstart-scala-SNAPSHOT.sh
@@ -30,7 +30,7 @@ mvn archetype:generate \
-Dversion=0.1 \
-Dpackage=org.apache.flink.quickstart \
-DinteractiveMode=false \
- -DarchetypeCatalog=https://oss.sonatype.org/content/repositories/snapshots/
+ -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/
#
# Give some guidance
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/dc55cc35/tools/deploy_to_maven.sh
----------------------------------------------------------------------
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index 619a172..78d1a32 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -46,20 +46,20 @@ function getVersion() {
# to the script (e.g. permissions re-evaled after suid)
exit 1 # fail
fi
- stratosphere_home="`dirname \"$here\"`"
- cd $stratosphere_home
+ flink_home="`dirname \"$here\"`"
+ cd $flink_home
echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)'`
}
# this will take a while
-CURRENT_STRATOSPHERE_VERSION=`getVersion`
-if [[ "$CURRENT_STRATOSPHERE_VERSION" == *-SNAPSHOT ]]; then
- CURRENT_STRATOSPHERE_VERSION_YARN=${CURRENT_STRATOSPHERE_VERSION/-SNAPSHOT/-hadoop2-SNAPSHOT}
+CURRENT_FLINK_VERSION=`getVersion`
+if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
+ CURRENT_FLINK_VERSION_YARN=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop2-SNAPSHOT}
else
- CURRENT_STRATOSPHERE_VERSION_YARN="$CURRENT_STRATOSPHERE_VERSION-hadoop2"
+ CURRENT_FLINK_VERSION_YARN="$CURRENT_FLINK_VERSION-hadoop2"
fi
-echo "detected current version as: '$CURRENT_STRATOSPHERE_VERSION' ; yarn: $CURRENT_STRATOSPHERE_VERSION_YARN "
+echo "detected current version as: '$CURRENT_FLINK_VERSION' ; yarn: $CURRENT_FLINK_VERSION_YARN "
# Check if push/commit is eligible for pushing
echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
@@ -70,24 +70,24 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
# It will deploy both a hadoop v1 and a hadoop v2 (yarn) artifact
#
- if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then
+ if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
# Deploy regular hadoop v1 to maven
- mvn -DskipTests deploy --settings deploysettings.xml;
+ mvn -DskipTests -Drat.ignoreErrors=true deploy --settings deploysettings.xml;
fi
- if [[ $TRAVIS_JOB_NUMBER == *4 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then
+ if [[ $TRAVIS_JOB_NUMBER == *4 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
# deploy hadoop v2 (yarn)
echo "Generating poms for hadoop-yarn."
- ./tools/generate_specific_pom.sh $CURRENT_STRATOSPHERE_VERSION $CURRENT_STRATOSPHERE_VERSION_YARN
+ ./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_YARN
# all these tweaks assume a yarn build.
# performance tweaks here: no "clean deploy" so that actually nothing is being rebuild (could cause wrong poms inside the jars?)
# skip tests (they were running already)
# skip javadocs generation (already generated)
- mvn -B -f pom.hadoop2.xml -DskipTests -Dmaven.javadoc.skip=true deploy --settings deploysettings.xml;
+ mvn -B -f pom.hadoop2.xml -DskipTests -Drat.ignoreErrors=true -Dmaven.javadoc.skip=true deploy --settings deploysettings.xml;
fi
- if [[ $TRAVIS_JOB_NUMBER == *5 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then
- cd stratosphere-java
+ if [[ $TRAVIS_JOB_NUMBER == *5 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then
+ cd flink-java
mvn javadoc:javadoc
cd target
cd apidocs
@@ -118,19 +118,19 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
if [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
#generate yarn poms & build for yarn.
# it is not required to generate poms for this build.
- #./tools/generate_specific_pom.sh $CURRENT_STRATOSPHERE_VERSION $CURRENT_STRATOSPHERE_VERSION_YARN pom.xml
+ #./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_YARN pom.xml
#mvn -B -DskipTests clean install
- CURRENT_STRATOSPHERE_VERSION=$CURRENT_STRATOSPHERE_VERSION_YARN
- YARN_ARCHIVE="stratosphere-dist/target/*yarn.tar.gz"
+ CURRENT_FLINK_VERSION=$CURRENT_FLINK_VERSION_YARN
+ YARN_ARCHIVE="flink-dist/target/*yarn.tar.gz"
fi
if [[ $TRAVIS_JOB_NUMBER == *3 ]] || [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
- # cd stratosphere-dist
+ # cd flink-dist
# mvn -B -DskipTests -Pdebian-package package
# cd ..
echo "Uploading build to amazon s3. Job Number: $TRAVIS_JOB_NUMBER"
- mkdir stratosphere
- cp -r stratosphere-dist/target/stratosphere-dist-*-bin/stratosphere*/* stratosphere/
- tar -czf stratosphere-$CURRENT_STRATOSPHERE_VERSION.tgz stratosphere
+ mkdir flink
+ cp -r flink-dist/target/flink-dist-*-bin/flink*/* flink/
+ tar -czf flink-$CURRENT_FLINK_VERSION.tgz flink
# upload the two in parallel
if [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
@@ -138,7 +138,7 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
mv $YARN_ARCHIVE .
travis-artifacts upload --path *yarn.tar.gz --target-path /
fi
- travis-artifacts upload --path stratosphere-$CURRENT_STRATOSPHERE_VERSION.tgz --target-path /
+ travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION.tgz --target-path /
fi
fi # pull request check
[21/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
index 265ce75..c0ea26a 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
@@ -22,12 +22,12 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
import org.apache.flink.api.java.DataSet;
@@ -134,7 +134,7 @@ public class EnumTrianglesOpt {
// *************************************************************************
/** Converts a Tuple2 into an Edge */
- public static class TupleEdgeConverter extends MapFunction<Tuple2<Integer, Integer>, Edge> {
+ public static class TupleEdgeConverter implements MapFunction<Tuple2<Integer, Integer>, Edge> {
private final Edge outEdge = new Edge();
@Override
@@ -145,7 +145,7 @@ public class EnumTrianglesOpt {
}
/** Emits for an edge the original edge and its switched version. */
- private static class EdgeDuplicator extends FlatMapFunction<Edge, Edge> {
+ private static class EdgeDuplicator implements FlatMapFunction<Edge, Edge> {
@Override
public void flatMap(Edge edge, Collector<Edge> out) throws Exception {
@@ -160,7 +160,7 @@ public class EnumTrianglesOpt {
* Emits one edge for each input edge with a degree annotation for the shared vertex.
* For each emitted edge, the first vertex is the vertex with the smaller id.
*/
- private static class DegreeCounter extends GroupReduceFunction<Edge, EdgeWithDegrees> {
+ private static class DegreeCounter implements GroupReduceFunction<Edge, EdgeWithDegrees> {
final ArrayList<Integer> otherVertices = new ArrayList<Integer>();
final EdgeWithDegrees outputEdge = new EdgeWithDegrees();
@@ -208,7 +208,7 @@ public class EnumTrianglesOpt {
* Builds an edge with degree annotation from two edges that have the same vertices and only one
* degree annotation.
*/
- private static class DegreeJoiner extends ReduceFunction<EdgeWithDegrees> {
+ private static class DegreeJoiner implements ReduceFunction<EdgeWithDegrees> {
private final EdgeWithDegrees outEdge = new EdgeWithDegrees();
@Override
@@ -228,7 +228,7 @@ public class EnumTrianglesOpt {
}
/** Projects an edge (pair of vertices) such that the first vertex is the vertex with the smaller degree. */
- private static class EdgeByDegreeProjector extends MapFunction<EdgeWithDegrees, Edge> {
+ private static class EdgeByDegreeProjector implements MapFunction<EdgeWithDegrees, Edge> {
private final Edge outEdge = new Edge();
@@ -249,7 +249,7 @@ public class EnumTrianglesOpt {
}
/** Projects an edge (pair of vertices) such that the id of the first is smaller than the id of the second. */
- private static class EdgeByIdProjector extends MapFunction<Edge, Edge> {
+ private static class EdgeByIdProjector implements MapFunction<Edge, Edge> {
@Override
public Edge map(Edge inEdge) throws Exception {
@@ -268,7 +268,7 @@ public class EnumTrianglesOpt {
* The first vertex of a triad is the shared vertex, the second and third vertex are ordered by vertexId.
* Assumes that input edges share the first vertex and are in ascending order of the second vertex.
*/
- private static class TriadBuilder extends GroupReduceFunction<Edge, Triad> {
+ private static class TriadBuilder implements GroupReduceFunction<Edge, Triad> {
private final List<Integer> vertices = new ArrayList<Integer>();
private final Triad outTriad = new Triad();
@@ -300,7 +300,7 @@ public class EnumTrianglesOpt {
}
/** Filters triads (three vertices connected by two edges) without a closing third edge. */
- private static class TriadFilter extends JoinFunction<Triad, Edge, Triad> {
+ private static class TriadFilter implements JoinFunction<Triad, Edge, Triad> {
@Override
public Triad join(Triad triad, Edge edge) throws Exception {
@@ -332,7 +332,7 @@ public class EnumTrianglesOpt {
System.out.println("Executing Enum Triangles Opt example with built-in default data.");
System.out.println(" Provide parameters to read input data from files.");
System.out.println(" See the documentation for the correct format of input files.");
- System.out.println(" Usage: EnumTriangleBasic <edge path> <result path>");
+ System.out.println(" Usage: EnumTriangleOpt <edge path> <result path>");
}
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
index 18eba5d..ba9754f 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
@@ -23,10 +23,10 @@ import static org.apache.flink.api.java.aggregation.Aggregations.SUM;
import java.util.ArrayList;
import java.util.Iterator;
-import org.apache.flink.api.java.functions.FilterFunction;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -139,7 +139,7 @@ public class PageRankBasic {
/**
* A map function that assigns an initial rank to all pages.
*/
- public static final class RankAssigner extends MapFunction<Tuple1<Long>, Tuple2<Long, Double>> {
+ public static final class RankAssigner implements MapFunction<Tuple1<Long>, Tuple2<Long, Double>> {
Tuple2<Long, Double> outPageWithRank;
public RankAssigner(double rank) {
@@ -158,7 +158,7 @@ public class PageRankBasic {
* originate. Run as a pre-processing step.
*/
@ConstantFields("0")
- public static final class BuildOutgoingEdgeList extends GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long[]>> {
+ public static final class BuildOutgoingEdgeList implements GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long[]>> {
private final ArrayList<Long> neighbors = new ArrayList<Long>();
@@ -179,7 +179,7 @@ public class PageRankBasic {
/**
* Join function that distributes a fraction of a vertex's rank to all neighbors.
*/
- public static final class JoinVertexWithEdgesMatch extends FlatMapFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>>, Tuple2<Long, Double>> {
+ public static final class JoinVertexWithEdgesMatch implements FlatMapFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>>, Tuple2<Long, Double>> {
@Override
public void flatMap(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>> value, Collector<Tuple2<Long, Double>> out){
@@ -197,7 +197,7 @@ public class PageRankBasic {
* The function that applies the page rank dampening formula
*/
@ConstantFields("0")
- public static final class Dampener extends MapFunction<Tuple2<Long,Double>, Tuple2<Long,Double>> {
+ public static final class Dampener implements MapFunction<Tuple2<Long,Double>, Tuple2<Long,Double>> {
private final double dampening;
private final double randomJump;
@@ -217,7 +217,7 @@ public class PageRankBasic {
/**
* Filter that filters vertices where the rank difference is below a threshold.
*/
- public static final class EpsilonFilter extends FilterFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>>> {
+ public static final class EpsilonFilter implements FilterFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>>> {
@Override
public boolean filter(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>> value) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
index d8d8b62..22054da 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
@@ -19,6 +19,8 @@
package org.apache.flink.example.java.graph;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
@@ -26,8 +28,6 @@ import org.apache.flink.api.java.IterativeDataSet;
import java.util.Iterator;
import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.example.java.graph.util.ConnectedComponentsData;
import org.apache.flink.util.Collector;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/ml/LinearRegression.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/ml/LinearRegression.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/ml/LinearRegression.java
index 1d687f3..0868732 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/ml/LinearRegression.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/ml/LinearRegression.java
@@ -22,8 +22,9 @@ package org.apache.flink.example.java.ml;
import java.io.Serializable;
import java.util.Collection;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.example.java.ml.util.LinearRegressionData;
@@ -183,7 +184,7 @@ public class LinearRegression {
// *************************************************************************
/** Converts a Tuple2<Double,Double> into a Data. */
- public static final class TupleDataConverter extends MapFunction<Tuple2<Double, Double>, Data> {
+ public static final class TupleDataConverter implements MapFunction<Tuple2<Double, Double>, Data> {
@Override
public Data map(Tuple2<Double, Double> t) throws Exception {
@@ -192,7 +193,7 @@ public class LinearRegression {
}
/** Converts a Tuple2<Double,Double> into a Params. */
- public static final class TupleParamsConverter extends MapFunction<Tuple2<Double, Double>,Params> {
+ public static final class TupleParamsConverter implements MapFunction<Tuple2<Double, Double>,Params> {
@Override
public Params map(Tuple2<Double, Double> t)throws Exception {
@@ -203,7 +204,7 @@ public class LinearRegression {
/**
* Compute a single BGD type update for every parameters.
*/
- public static class SubUpdate extends MapFunction<Data,Tuple2<Params,Integer>>{
+ public static class SubUpdate extends RichMapFunction<Data,Tuple2<Params,Integer>> {
private Collection<Params> parameters;
@@ -234,7 +235,7 @@ public class LinearRegression {
/**
* Accumulator all the update.
* */
- public static class UpdateAccumulator extends ReduceFunction<Tuple2<Params, Integer>> {
+ public static class UpdateAccumulator implements ReduceFunction<Tuple2<Params, Integer>> {
@Override
public Tuple2<Params, Integer> reduce(Tuple2<Params, Integer> val1, Tuple2<Params, Integer> val2) {
@@ -250,7 +251,7 @@ public class LinearRegression {
/**
* Compute the final update by average them.
*/
- public static class Update extends MapFunction<Tuple2<Params, Integer>,Params>{
+ public static class Update implements MapFunction<Tuple2<Params, Integer>,Params> {
@Override
public Params map(Tuple2<Params, Integer> arg0) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/EmptyFieldsCountAccumulator.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/EmptyFieldsCountAccumulator.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/EmptyFieldsCountAccumulator.java
index 61b90dd..4bced17 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/EmptyFieldsCountAccumulator.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/EmptyFieldsCountAccumulator.java
@@ -26,7 +26,7 @@ import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.FilterFunction;
+import org.apache.flink.api.java.functions.RichFilterFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
@@ -149,7 +149,7 @@ public class EmptyFieldsCountAccumulator {
* In doing so, it also counts the number of empty fields per attribute with an accumulator (registered under
* {@link EmptyFieldsCountAccumulator#EMPTY_FIELD_ACCUMULATOR}).
*/
- public static final class EmptyFieldFilter extends FilterFunction<Tuple> {
+ public static final class EmptyFieldFilter extends RichFilterFunction<Tuple> {
// create a new accumulator in each filter function instance
// accumulators can be merged later on
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/RelationalQuery.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/RelationalQuery.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/RelationalQuery.java
index 48cdedc..08a261c 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/RelationalQuery.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/RelationalQuery.java
@@ -18,8 +18,8 @@
package org.apache.flink.example.java.relational;
+import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FilterFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery10.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery10.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery10.java
index ef03e6f..1ff6583 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery10.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery10.java
@@ -18,9 +18,9 @@
package org.apache.flink.example.java.relational;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FilterFunction;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java
index 52109ea..4544fd4 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java
@@ -25,9 +25,9 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FilterFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
index 2649d24..3033c0d 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
@@ -21,8 +21,8 @@ package org.apache.flink.example.java.relational;
import java.util.Iterator;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.FilterFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
@@ -159,7 +159,7 @@ public class WebLogAnalysis {
* MapFunction that filters for documents that contain a certain set of
* keywords.
*/
- public static class FilterDocByKeyWords extends FilterFunction<Tuple2<String, String>> {
+ public static class FilterDocByKeyWords implements FilterFunction<Tuple2<String, String>> {
private static final String[] KEYWORDS = { " editors ", " oscillations " };
@@ -187,7 +187,7 @@ public class WebLogAnalysis {
/**
* MapFunction that filters for records where the rank exceeds a certain threshold.
*/
- public static class FilterByRank extends FilterFunction<Tuple3<Integer, String, Integer>> {
+ public static class FilterByRank implements FilterFunction<Tuple3<Integer, String, Integer>> {
private static final int RANKFILTER = 40;
@@ -210,7 +210,7 @@ public class WebLogAnalysis {
* MapFunction that filters for records of the visits relation where the year
* (from the date string) is equal to a certain value.
*/
- public static class FilterVisitsByDate extends FilterFunction<Tuple2<String, String>> {
+ public static class FilterVisitsByDate implements FilterFunction<Tuple2<String, String>> {
private static final int YEARFILTER = 2007;
@@ -237,7 +237,7 @@ public class WebLogAnalysis {
* If the first input does not provide any pairs, all pairs of the second input are emitted.
* Otherwise, no pair is emitted.
*/
- public static class AntiJoinVisits extends CoGroupFunction<Tuple3<Integer, String, Integer>, Tuple1<String>, Tuple3<Integer, String, Integer>> {
+ public static class AntiJoinVisits implements CoGroupFunction<Tuple3<Integer, String, Integer>, Tuple1<String>, Tuple3<Integer, String, Integer>> {
/**
* If the visit iterator is empty, all pairs of the rank iterator are emitted.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCount.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCount.java
index a18abcb..3e95ccd 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCount.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCount.java
@@ -18,7 +18,7 @@
package org.apache.flink.example.java.wordcount;
-import org.apache.flink.api.java.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
import org.apache.flink.api.java.DataSet;
@@ -91,7 +91,7 @@ public class WordCount {
* FlatMapFunction. The function takes a line (String) and splits it into
* multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
*/
- public static final class Tokenizer extends FlatMapFunction<String, Tuple2<String, Integer>> {
+ public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
index 32a8997..04810a1 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
@@ -18,8 +18,8 @@
package org.apache.flink.example.java.wordcount;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -111,7 +111,7 @@ public class WordCountPOJO {
* FlatMapFunction. The function takes a line (String) and splits it into
* multiple WC POJOs as "(word, 1)".
*/
- public static final class Tokenizer extends FlatMapFunction<String, WC> {
+ public static final class Tokenizer implements FlatMapFunction<String, WC> {
@Override
public void flatMap(String value, Collector<WC> out) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/pom.xml b/flink-examples/pom.xml
index 917461a..ae46681 100644
--- a/flink-examples/pom.xml
+++ b/flink-examples/pom.xml
@@ -50,5 +50,4 @@ under the License.
<module>flink-java-examples</module>
<module>flink-scala-examples</module>
</modules>
-
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index 894880e..e7199f9 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -19,42 +19,39 @@
package org.apache.flink.api.java;
import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.io.FileOutputFormat;
import org.apache.flink.api.common.io.OutputFormat;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.FilterFunction;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
import org.apache.flink.api.java.io.CsvOutputFormat;
import org.apache.flink.api.java.io.PrintingOutputFormat;
import org.apache.flink.api.java.io.TextOutputFormat;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.operators.CoGroupOperator;
+import org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets;
import org.apache.flink.api.java.operators.CrossOperator;
+import org.apache.flink.api.java.operators.CrossOperator.DefaultCross;
import org.apache.flink.api.java.operators.CustomUnaryOperation;
import org.apache.flink.api.java.operators.DataSink;
import org.apache.flink.api.java.operators.DistinctOperator;
import org.apache.flink.api.java.operators.FilterOperator;
import org.apache.flink.api.java.operators.FlatMapOperator;
-import org.apache.flink.api.java.operators.Grouping;
-import org.apache.flink.api.java.operators.JoinOperator;
+import org.apache.flink.api.java.operators.JoinOperator.JoinHint;
+import org.apache.flink.api.java.operators.JoinOperator.JoinOperatorSets;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.operators.MapOperator;
-import org.apache.flink.api.java.operators.ProjectOperator;
-import org.apache.flink.api.java.operators.ReduceGroupOperator;
+import org.apache.flink.api.java.operators.ProjectOperator.Projection;
+import org.apache.flink.api.java.operators.GroupReduceOperator;
import org.apache.flink.api.java.operators.ReduceOperator;
-import org.apache.flink.api.java.operators.SortedGrouping;
import org.apache.flink.api.java.operators.UnionOperator;
import org.apache.flink.api.java.operators.UnsortedGrouping;
-import org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets;
-import org.apache.flink.api.java.operators.CrossOperator.DefaultCross;
-import org.apache.flink.api.java.operators.JoinOperator.JoinHint;
-import org.apache.flink.api.java.operators.JoinOperator.JoinOperatorSets;
-import org.apache.flink.api.java.operators.ProjectOperator.Projection;
import org.apache.flink.api.java.record.functions.CrossFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -68,8 +65,8 @@ import org.apache.flink.types.TypeInformation;
* A DataSet represents a collection of elements of the same type.<br/>
* A DataSet can be transformed into another DataSet by applying a transformation as for example
* <ul>
- * <li>{@link DataSet#map(MapFunction)},</li>
- * <li>{@link DataSet#reduce(ReduceFunction)},</li>
+ * <li>{@link DataSet#map(org.apache.flink.api.java.functions.RichMapFunction)},</li>
+ * <li>{@link DataSet#reduce(org.apache.flink.api.java.functions.RichReduceFunction)},</li>
* <li>{@link DataSet#join(DataSet)}, or</li>
* <li>{@link DataSet#coGroup(DataSet)}.</li>
* </ul>
@@ -124,13 +121,13 @@ public abstract class DataSet<T> {
/**
* Applies a Map transformation on a {@link DataSet}.<br/>
- * The transformation calls a {@link MapFunction} for each element of the DataSet.
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichMapFunction} for each element of the DataSet.
* Each MapFunction call returns exactly one element.
*
* @param mapper The MapFunction that is called for each element of the DataSet.
* @return A MapOperator that represents the transformed DataSet.
*
- * @see MapFunction
+ * @see org.apache.flink.api.java.functions.RichMapFunction
* @see MapOperator
* @see DataSet
*/
@@ -138,18 +135,21 @@ public abstract class DataSet<T> {
if (mapper == null) {
throw new NullPointerException("Map function must not be null.");
}
+ if (FunctionUtils.isSerializedLambdaFunction(mapper)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
return new MapOperator<T, R>(this, mapper);
}
/**
* Applies a FlatMap transformation on a {@link DataSet}.<br/>
- * The transformation calls a {@link FlatMapFunction} for each element of the DataSet.
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichFlatMapFunction} for each element of the DataSet.
* Each FlatMapFunction call can return any number of elements including none.
*
* @param flatMapper The FlatMapFunction that is called for each element of the DataSet.
* @return A FlatMapOperator that represents the transformed DataSet.
*
- * @see FlatMapFunction
+ * @see org.apache.flink.api.java.functions.RichFlatMapFunction
* @see FlatMapOperator
* @see DataSet
*/
@@ -157,19 +157,22 @@ public abstract class DataSet<T> {
if (flatMapper == null) {
throw new NullPointerException("FlatMap function must not be null.");
}
+ if (FunctionUtils.isSerializedLambdaFunction(flatMapper)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
return new FlatMapOperator<T, R>(this, flatMapper);
}
/**
* Applies a Filter transformation on a {@link DataSet}.<br/>
- * The transformation calls a {@link FilterFunction} for each element of the DataSet
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichFilterFunction} for each element of the DataSet
* and retains only those element for which the function returns true. Elements for
* which the function returns false are filtered.
*
* @param filter The FilterFunction that is called for each element of the DataSet.
* @return A FilterOperator that represents the filtered DataSet.
*
- * @see FilterFunction
+ * @see org.apache.flink.api.java.functions.RichFilterFunction
* @see FilterOperator
* @see DataSet
*/
@@ -179,6 +182,7 @@ public abstract class DataSet<T> {
}
return new FilterOperator<T>(this, filter);
}
+
// --------------------------------------------------------------------------------------------
// Projections
@@ -264,14 +268,14 @@ public abstract class DataSet<T> {
/**
* Applies a Reduce transformation on a non-grouped {@link DataSet}.<br/>
- * The transformation consecutively calls a {@link ReduceFunction}
+ * The transformation consecutively calls a {@link org.apache.flink.api.java.functions.RichReduceFunction}
* until only a single element remains which is the result of the transformation.
* A ReduceFunction combines two elements into one new element of the same type.
*
* @param reducer The ReduceFunction that is applied on the DataSet.
* @return A ReduceOperator that represents the reduced DataSet.
*
- * @see ReduceFunction
+ * @see org.apache.flink.api.java.functions.RichReduceFunction
* @see ReduceOperator
* @see DataSet
*/
@@ -284,24 +288,27 @@ public abstract class DataSet<T> {
/**
* Applies a GroupReduce transformation on a non-grouped {@link DataSet}.<br/>
- * The transformation calls a {@link GroupReduceFunction} once with the full DataSet.
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichGroupReduceFunction} once with the full DataSet.
* The GroupReduceFunction can iterate over all elements of the DataSet and emit any
* number of output elements including none.
*
* @param reducer The GroupReduceFunction that is applied on the DataSet.
* @return A GroupReduceOperator that represents the reduced DataSet.
*
- * @see GroupReduceFunction
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.functions.RichGroupReduceFunction
+ * @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
- public <R> ReduceGroupOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
+ public <R> GroupReduceOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- return new ReduceGroupOperator<T, R>(this, reducer);
+ if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
+ return new GroupReduceOperator<T, R>(this, reducer);
}
-
+
// --------------------------------------------------------------------------------------------
// distinct
// --------------------------------------------------------------------------------------------
@@ -359,8 +366,8 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(ReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(GroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param keyExtractor The KeySelector function which extracts the key values from the DataSet on which it is grouped.
@@ -372,7 +379,7 @@ public abstract class DataSet<T> {
* @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
public <K extends Comparable<K>> UnsortedGrouping<T> groupBy(KeySelector<T, K> keyExtractor) {
@@ -388,8 +395,8 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(ReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(GroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param fields One or more field positions on which the DataSet will be grouped.
@@ -401,7 +408,7 @@ public abstract class DataSet<T> {
* @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
public UnsortedGrouping<T> groupBy(int... fields) {
@@ -417,8 +424,8 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(ReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(GroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param fields One or more field expressions on which the DataSet will be grouped.
@@ -430,7 +437,7 @@ public abstract class DataSet<T> {
* @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
public UnsortedGrouping<T> groupBy(String... fields) {
@@ -461,7 +468,7 @@ public abstract class DataSet<T> {
public <R> JoinOperatorSets<T, R> join(DataSet<R> other) {
return new JoinOperatorSets<T, R>(this, other);
}
-
+
/**
* Initiates a Join transformation. <br/>
* A Join transformation joins the elements of two
@@ -514,7 +521,7 @@ public abstract class DataSet<T> {
* Initiates a CoGroup transformation.<br/>
* A CoGroup transformation combines the elements of
* two {@link DataSet DataSets} into one DataSet. It groups each DataSet individually on a key and
- * gives groups of both DataSets with equal keys together into a {@link CoGroupFunction}.
+ * gives groups of both DataSets with equal keys together into a {@link org.apache.flink.api.java.functions.RichCoGroupFunction}.
* If a DataSet has a group with no matching key in the other DataSet, the CoGroupFunction
* is called with an empty group for the non-existing group.</br>
* The CoGroupFunction can iterate over the elements of both groups and return any number
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/DeltaIteration.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DeltaIteration.java b/flink-java/src/main/java/org/apache/flink/api/java/DeltaIteration.java
index bb53a89..2f2eae0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DeltaIteration.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DeltaIteration.java
@@ -189,7 +189,7 @@ public class DeltaIteration<ST, WT> {
* The value of an aggregator can be accessed in the next iteration.
* <p>
* Aggregators can be accessed inside a function via the
- * {@link org.apache.flink.api.common.functions.AbstractFunction#getIterationRuntimeContext()} method.
+ * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getIterationRuntimeContext()} method.
*
* @param name The name under which the aggregator is registered.
* @param aggregator The aggregator class.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index d00fb47..ebd1422 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -112,7 +112,7 @@ public abstract class ExecutionEnvironment {
* individually override this value to use a specific degree of parallelism via
* {@link Operator#setParallelism(int)}. Other operations may need to run with a different
* degree of parallelism - for example calling
- * {@link DataSet#reduce(org.apache.flink.api.java.functions.ReduceFunction)} over the entire
+ * {@link DataSet#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} over the entire
* set will insert eventually an operation that runs non-parallel (degree of parallelism of one).
*
* @return The degree of parallelism used by operations, unless they override that value. This method
@@ -550,7 +550,7 @@ public abstract class ExecutionEnvironment {
* The runtime will copy the files temporarily to a local cache, if needed.
* <p>
* The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
- * {@link org.apache.flink.api.common.functions.Function#getRuntimeContext()} and provides access
+ * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
* {@link org.apache.flink.api.common.cache.DistributedCache} via
* {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
*
@@ -568,7 +568,7 @@ public abstract class ExecutionEnvironment {
* The runtime will copy the files temporarily to a local cache, if needed.
* <p>
* The {@link org.apache.flink.api.common.functions.RuntimeContext} can be obtained inside UDFs via
- * {@link org.apache.flink.api.common.functions.Function#getRuntimeContext()} and provides access
+ * {@link org.apache.flink.api.common.functions.RichFunction#getRuntimeContext()} and provides access
* {@link org.apache.flink.api.common.cache.DistributedCache} via
* {@link org.apache.flink.api.common.functions.RuntimeContext#getDistributedCache()}.
*
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/IterativeDataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/IterativeDataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/IterativeDataSet.java
index 6443dbb..5dc0d2e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/IterativeDataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/IterativeDataSet.java
@@ -93,7 +93,7 @@ public class IterativeDataSet<T> extends SingleInputOperator<T, T, IterativeData
* The value of an aggregator can be accessed in the next iteration.
* <p>
* Aggregators can be accessed inside a function via the
- * {@link org.apache.flink.api.common.functions.AbstractFunction#getIterationRuntimeContext()} method.
+ * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getIterationRuntimeContext()} method.
*
* @param name The name under which the aggregator is registered.
* @param aggregator The aggregator class.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/CoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/CoGroupFunction.java
deleted file mode 100644
index 201794a..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/CoGroupFunction.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import java.util.Iterator;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
-import org.apache.flink.util.Collector;
-
-/**
- * The abstract base class for CoGroup functions. CoGroup functions combine two data sets by first grouping each data set
- * after a key and then "joining" the groups by calling this function with the two sets for each key.
- * If a key is present in only one of the two inputs, it may be that one of the groups is empty.
- * <p>
- * The basic syntax for using CoGoup on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.coGroup(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyCoGroupFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * The keys can be defined through tuple field positions or key extractors.
- * See {@link org.apache.flink.api.java.operators.Keys} for details.
- * <p>
- * Some keys may only be contained in one of the two original data sets. In that case, the CoGroup function is invoked
- * with in empty input for the side of the data set that did not contain elements with that specific key.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN1> The type of the elements in the first input.
- * @param <IN2> The type of the elements in the second input.
- * @param <OUT> The type of the result elements.
- */
-public abstract class CoGroupFunction<IN1, IN2, OUT> extends AbstractFunction implements GenericCoGrouper<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-
-
- /**
- * The core method of the CoGroupFunction. This method is called for each pair of groups that have the same
- * key. The elements of the groups are returned by the respective iterators.
- *
- * It is possible that one of the two groups is empty, in which case the respective iterator has no elements.
- *
- * @param first The group from the first input.
- * @param second The group from the second input.
- * @param out The collector through which to return the result elements.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public abstract void coGroup(Iterator<IN1> first, Iterator<IN2> second, Collector<OUT> out) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/CrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/CrossFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/CrossFunction.java
deleted file mode 100644
index 27907ec..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/CrossFunction.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.util.Collector;
-
-
-/**
- * The abstract base class for Cross functions. Cross functions build a Cartesian produce of their inputs
- * and call the function or each pair of elements.
- * They are a means of convenience and can be used to directly produce manipulate the
- * pair of elements, instead of having the operator build 2-tuples, and then using a
- * MapFunction over those 2-tuples.
- * <p>
- * The basic syntax for using Cross on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.cross(set2).with(new MyCrossFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN1> The type of the elements in the first input.
- * @param <IN2> The type of the elements in the second input.
- * @param <OUT> The type of the result elements.
- */
-public abstract class CrossFunction<IN1, IN2, OUT> extends AbstractFunction implements GenericCrosser<IN1, IN2, OUT>{
-
- private static final long serialVersionUID = 1L;
-
-
- /**
- * The core method of the cross operation. The method will be invoked for each pair of elements
- * in the Cartesian product.
- *
- * @param first The element from the first input.
- * @param second The element from the second input.
- * @return The result element.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- public abstract OUT cross(IN1 first, IN2 second) throws Exception;
-
-
-
- /**
- * This method only delegates calls to the {@link #cross(Object, Object)} method.
- */
- @Override
- public final void cross(IN1 record1, IN2 record2, Collector<OUT> out) throws Exception {
- out.collect(cross(record1, record2));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/FilterFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/FilterFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/FilterFunction.java
deleted file mode 100644
index aac2086..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/FilterFunction.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericFilter;
-
-/**
- * The abstract base class for Filter functions. A filter function take elements and evaluates a
- * predicate on them to decide whether to keep the element, or to discard it.
- * <p>
- * The basic syntax for using a FilterFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.filter(new MyFilterFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the FilterFunction needs to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <T> The type of the filtered elements.
- */
-public abstract class FilterFunction<T> extends AbstractFunction implements GenericFilter<T> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the FilterFunction. The method is called for each element in the input,
- * and determines whether the element should be kept or filtered out. If the method returns true,
- * the element passes the filter and is kept, if the method returns false, the element is
- * filtered out.
- *
- * @param value The input value to be filtered.
- * @return Flag to indicate whether to keep the value (true) or to discard it (false).
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public abstract boolean filter(T value) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapFunction.java
deleted file mode 100644
index f9c22cc..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapFunction.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericFlatMap;
-import org.apache.flink.util.Collector;
-
-/**
- * The abstract base class for flatMap functions. FlatMap functions take elements and transform them,
- * into zero, one, or more elements. Typical applications can be splitting elements, or unnesting lists
- * and arrays. Operations that produce multiple strictly one result element per input element can also
- * use the {@link MapFunction}.
- * <p>
- * The basic syntax for using a FlatMapFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<Y> result = input.flatMap(new MyFlatMapFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the FlatMapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN> Type of the input elements.
- * @param <OUT> Type of the returned elements.
- */
-public abstract class FlatMapFunction<IN, OUT> extends AbstractFunction implements GenericFlatMap<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the FlatMapFunction. Takes an element from the input data set and transforms
- * it into zero, one, or more elements.
- *
- * @param value The input value.
- * @param out The collector for for emitting result values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public abstract void flatMap(IN value, Collector<OUT> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
index 5cc8c12..012ab57 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
@@ -23,7 +23,7 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
/**
- * A variant of the {@link FlatMapFunction} that returns elements through an iterator, rather then
+ * A variant of the {@link RichFlatMapFunction} that returns elements through an iterator, rather then
* through a collector. In all other respects, it behaves exactly like the FlatMapFunction.
* <p>
* The function needs to be serializable, as defined in {@link java.io.Serializable}.
@@ -31,7 +31,7 @@ import org.apache.flink.util.Collector;
* @param <IN> Type of the input elements.
* @param <OUT> Type of the returned elements.
*/
-public abstract class FlatMapIterator<IN, OUT> extends FlatMapFunction<IN, OUT> {
+public abstract class FlatMapIterator<IN, OUT> extends RichFlatMapFunction<IN, OUT> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/FunctionAnnotation.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/FunctionAnnotation.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/FunctionAnnotation.java
index 1015971..b94840f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/FunctionAnnotation.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/FunctionAnnotation.java
@@ -64,15 +64,15 @@ import org.apache.flink.api.common.InvalidProgramException;
* </b>
* <p>
* Be aware that some annotations should only be used for functions with as single input
- * ({@link MapFunction}, {@link ReduceFunction}) and some only for stubs with two inputs
- * ({@link CrossFunction}, {@link JoinFunction}, {@link CoGroupFunction}).
+ * ({@link RichMapFunction}, {@link RichReduceFunction}) and some only for stubs with two inputs
+ * ({@link RichCrossFunction}, {@link RichFlatJoinFunction}, {@link RichCoGroupFunction}).
*/
public class FunctionAnnotation {
/**
* This annotation declares that a function leaves certain fields of its input values unmodified and
* only "forwards" or "copies" them to the return value. The annotation is applicable to unary
- * functions, like for example {@link MapFunction}, {@link ReduceFunction}, or {@link FlatMapFunction}.
+ * functions, like for example {@link RichMapFunction}, {@link RichReduceFunction}, or {@link RichFlatMapFunction}.
* <p>
* The following example illustrates a function that keeps the tuple's field zero constant:
* <pre><blockquote>
@@ -103,7 +103,7 @@ public class FunctionAnnotation {
/**
* This annotation declares that a function leaves certain fields of its first input values unmodified and
* only "forwards" or "copies" them to the return value. The annotation is applicable to binary
- * functions, like for example {@link JoinFunction}, {@link CoGroupFunction}, or {@link CrossFunction}.
+ * functions, like for example {@link RichFlatJoinFunction}, {@link RichCoGroupFunction}, or {@link RichCrossFunction}.
* <p>
* The following example illustrates a join function that copies fields from the first and second input to the
* return value:
@@ -135,7 +135,7 @@ public class FunctionAnnotation {
/**
* This annotation declares that a function leaves certain fields of its second input values unmodified and
* only "forwards" or "copies" them to the return value. The annotation is applicable to binary
- * functions, like for example {@link JoinFunction}, {@link CoGroupFunction}, or {@link CrossFunction}.
+ * functions, like for example {@link RichFlatJoinFunction}, {@link RichCoGroupFunction}, or {@link RichCrossFunction}.
* <p>
* The following example illustrates a join function that copies fields from the first and second input to the
* return value:
@@ -167,7 +167,7 @@ public class FunctionAnnotation {
/**
* This annotation declares that a function changes certain fields of its input values, while leaving all
* others unmodified and in place in the return value. The annotation is applicable to unary
- * functions, like for example {@link MapFunction}, {@link ReduceFunction}, or {@link FlatMapFunction}.
+ * functions, like for example {@link RichMapFunction}, {@link RichReduceFunction}, or {@link RichFlatMapFunction}.
* <p>
* The following example illustrates that at the example of a Map function:
*
@@ -201,7 +201,7 @@ public class FunctionAnnotation {
/**
* This annotation declares that a function changes certain fields of its first input value, while leaving all
* others unmodified and in place in the return value. The annotation is applicable to binary
- * functions, like for example {@link JoinFunction}, {@link CoGroupFunction}, or {@link CrossFunction}.
+ * functions, like for example {@link RichFlatJoinFunction}, {@link RichCoGroupFunction}, or {@link RichCrossFunction}.
* <p>
* The following example illustrates a join function that copies fields from the first and second input to the
* return value:
@@ -238,7 +238,7 @@ public class FunctionAnnotation {
/**
* This annotation declares that a function changes certain fields of its second input value, while leaving all
* others unmodified and in place in the return value. The annotation is applicable to binary
- * functions, like for example {@link JoinFunction}, {@link CoGroupFunction}, or {@link CrossFunction}.
+ * functions, like for example {@link RichFlatJoinFunction}, {@link RichCoGroupFunction}, or {@link RichCrossFunction}.
* <p>
* The following example illustrates a join function that copies fields from the first and second input to the
* return value:
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceFunction.java
deleted file mode 100644
index 01ae9c1..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceFunction.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.Iterator;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.util.Collector;
-
-/**
- * The abstract base class for group reduce functions. Group reduce functions process groups of elements.
- * They may aggregate them to a single value, or produce multiple result values for each group.
- * <p>
- * For a reduce functions that works incrementally by combining always two elements, see
- * {@link ReduceFunction}, called via {@link org.apache.flink.api.java.DataSet#reduce(ReduceFunction)}.
- * <p>
- * The basic syntax for using a grouped GroupReduceFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.groupBy(<key-definition>).reduceGroup(new MyGroupReduceFunction());
- * </blockquote></pre>
- * <p>
- * GroupReduceFunctions may be "combinable", in which case they can pre-reduce partial groups in order to
- * reduce the data volume early. See the {@link #combine(Iterator, Collector)} function for details.
- * <p>
- * Like all functions, the GroupReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN> Type of the elements that this function processes.
- * @param <OUT> The type of the elements returned by the user-defined function.
- */
-public abstract class GroupReduceFunction<IN, OUT> extends AbstractFunction implements GenericGroupReduce<IN, OUT>, GenericCombine<IN> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Core method of the reduce function. It is called one per group of elements. If the reducer
- * is not grouped, than the entire data set is considered one group.
- *
- * @param values The iterator returning the group of values to be reduced.
- * @param out The collector to emit the returned values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public abstract void reduce(Iterator<IN> values, Collector<OUT> out) throws Exception;
-
- /**
- * The combine methods pre-reduces elements. It may be called on subsets of the data
- * before the actual reduce function. This is often helpful to lower data volume prior
- * to reorganizing the data in an expensive way, as might be required for the final
- * reduce function.
- * <p>
- * This method is only ever invoked when the subclass of {@link GroupReduceFunction}
- * adds the {@link Combinable} annotation, or if the <i>combinable</i> flag is set when defining
- * the <i>reduceGroup<i> operation via
- * {@link org.apache.flink.api.java.operators.ReduceGroupOperator#setCombinable(boolean)}.
- * <p>
- * Since the reduce function will be called on the result of this method, it is important that this
- * method returns the same data type as it consumes. By default, this method only calls the
- * {@link #reduce(Iterator, Collector)} method. If the behavior in the pre-reducing is different
- * from the final reduce function (for example because the reduce function changes the data type),
- * this method must be overwritten, or the execution will fail.
- *
- * @param values The iterator returning the group of values to be reduced.
- * @param out The collector to emit the returned values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public void combine(Iterator<IN> values, Collector<IN> out) throws Exception {
- @SuppressWarnings("unchecked")
- Collector<OUT> c = (Collector<OUT>) out;
- reduce(values, c);
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * This annotation can be added to classes that extend {@link GroupReduceFunction}, in oder to mark
- * them as "combinable". The system may call the {@link GroupReduceFunction#combine(Iterator, Collector)}
- * method on such functions, to pre-reduce the data before transferring it over the network to
- * the actual group reduce operation.
- * <p>
- * Marking combinable functions as such is in general beneficial for performance.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target(ElementType.TYPE)
- public static @interface Combinable {};
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
index 6cb397b..b363606 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
@@ -23,7 +23,7 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
-public abstract class GroupReduceIterator<IN, OUT> extends GroupReduceFunction<IN, OUT> {
+public abstract class GroupReduceIterator<IN, OUT> extends RichGroupReduceFunction<IN, OUT> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/JoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/JoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/JoinFunction.java
deleted file mode 100644
index c78e6f3..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/JoinFunction.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericJoiner;
-import org.apache.flink.util.Collector;
-
-/**
- * The abstract base class for Join functions. Join functions combine two data sets by joining their
- * elements on specified keys and calling this function with each pair of joining elements.
- * By default, this follows strictly the semantics of an "inner join" in SQL.
- * the semantics are those of an "inner join", meaning that elements are filtered out
- * if their key is not contained in the other data set.
- * <p>
- * Per the semantics of an inner join, the function is
- * <p>
- * The basic syntax for using Join on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.join(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyJoinFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * The keys can be defined through tuple field positions or key extractors.
- * See {@link org.apache.flink.api.java.operators.Keys} for details.
- * <p>
- * The Join function is actually not a necessary part of a join operation. If no JoinFunction is provided,
- * the result of the operation is a sequence of Tuple2, where the elements in the tuple are those that
- * the JoinFunction would have been invoked with.
- * <P>
- * Note: You can use a {@link CoGroupFunction} to perform an outer join.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN1> The type of the elements in the first input.
- * @param <IN2> The type of the elements in the second input.
- * @param <OUT> The type of the result elements.
- */
-public abstract class JoinFunction<IN1, IN2, OUT> extends AbstractFunction implements GenericJoiner<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The user-defined method for performing transformations after a join.
- * The method is called with matching pairs of elements from the inputs.
- *
- * @param first The element from first input.
- * @param second The element from second input.
- * @return The resulting element.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- public abstract OUT join(IN1 first, IN2 second) throws Exception;
-
-
- /**
- * The user-defined method for performing transformations after a join, for operations that
- * produce zero elements, or more than one element.
- * By default, this method delegates to the method {@link #join(Object, Object)}. If this method
- * is overridden, that method will no longer be called.
- *
- * @param value1 The element from first input.
- * @param value2 The element from second input.
- * @param out A collector to emit resulting element (zero, one, or many).
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public void join(IN1 value1, IN2 value2, Collector<OUT> out) throws Exception {
- out.collect(join(value1, value2));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/MapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/MapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/MapFunction.java
deleted file mode 100644
index 64aec2a..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/MapFunction.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericMap;
-
-/**
- * The abstract base class for Map functions. Map functions take elements and transform them,
- * element wise. A Map function always produces a single result element for each input element.
- * Typical applications are parsing elements, converting data types, or projecting out fields.
- * Operations that produce multiple result elements from a single input element can be implemented
- * using the {@link FlatMapFunction}.
- * <p>
- * The basic syntax for using a MapFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<Y> result = input.map(new MyMapFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the MapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <IN> Type of the input elements.
- * @param <OUT> Type of the returned elements.
- */
-public abstract class MapFunction<IN, OUT> extends AbstractFunction implements GenericMap<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the MapFunction. Takes an element from the input data set and transforms
- * it into another element.
- *
- * @param value The input value.
- * @return The value produced by the map function from the input value.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- @Override
- public abstract OUT map(IN value) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/ReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/ReduceFunction.java
deleted file mode 100644
index aea6bf8..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/ReduceFunction.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.functions;
-
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericReduce;
-
-/**
- * The abstract base class for Reduce functions. Reduce functions combine groups of elements to
- * a single value, by taking always two elements and combining them into one. Reduce functions
- * may be used on entire data sets, or on grouped data sets. In the latter case, each group is reduced
- * individually.
- * <p>
- * For a reduce functions that work on an entire group at the same time (such as the
- * MapReduce/Hadoop-style reduce), see {@link GroupReduceFunction}, called via
- * {@link org.apache.flink.api.java.DataSet#reduceGroup(GroupReduceFunction)}. In the general case,
- * ReduceFunctions are considered faster, because they allow the system to use hash-based
- * execution strategies.
- * <p>
- * The basic syntax for using a grouped ReduceFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.groupBy(<key-definition>).reduce(new MyReduceFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the ReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
- *
- * @param <T> Type of the elements that this function processes.
- */
-public abstract class ReduceFunction<T> extends AbstractFunction implements GenericReduce<T> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the ReduceFunction, combining two values into one value of the same type.
- * The reduce function is consecutively applied to all values of a group until only a single value remains.
- *
- * @param value1 The first value to combine.
- * @param value2 The second value to combine.
- * @return The combined value of both input values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
- public abstract T reduce(T value1, T value2) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
new file mode 100644
index 0000000..8aaaf86
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import java.util.Iterator;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.util.Collector;
+
+/**
+ * The abstract base class for CoGroup functions. CoGroup functions combine two data sets by first grouping each data set
+ * after a key and then "joining" the groups by calling this function with the two sets for each key.
+ * If a key is present in only one of the two inputs, it may be that one of the groups is empty.
+ * <p>
+ * The basic syntax for using CoGoup on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.coGroup(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyCoGroupFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * The keys can be defined through tuple field positions or key extractors.
+ * See {@link org.apache.flink.api.java.operators.Keys} for details.
+ * <p>
+ * Some keys may only be contained in one of the two original data sets. In that case, the CoGroup function is invoked
+ * with in empty input for the side of the data set that did not contain elements with that specific key.
+ * <p>
+ * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
+public abstract class RichCoGroupFunction<IN1, IN2, OUT> extends AbstractRichFunction implements CoGroupFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+
+ /**
+ * The core method of the CoGroupFunction. This method is called for each pair of groups that have the same
+ * key. The elements of the groups are returned by the respective iterators.
+ *
+ * It is possible that one of the two groups is empty, in which case the respective iterator has no elements.
+ *
+ * @param first The group from the first input.
+ * @param second The group from the second input.
+ * @param out The collector through which to return the result elements.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract void coGroup(Iterator<IN1> first, Iterator<IN2> second, Collector<OUT> out) throws Exception;
+
+}
[09/39] git commit: Update Netty dependency to 4.0.21.Final
Posted by rm...@apache.org.
Update Netty dependency to 4.0.21.Final
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/a45c14b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/a45c14b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/a45c14b6
Branch: refs/heads/travis_test
Commit: a45c14b6ddc236f0df62aaad37bb31b9ad8f211c
Parents: 5251263
Author: uce <u....@fu-berlin.de>
Authored: Tue Jul 29 17:42:13 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Wed Jul 30 01:03:25 2014 +0200
----------------------------------------------------------------------
flink-runtime/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a45c14b6/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 487beca..7bb49f4 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -75,7 +75,7 @@ under the License.
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
- <version>4.0.20.Final</version>
+ <version>4.0.21.Final</version>
</dependency>
<dependency>
[10/39] git commit: Add type parameters in OutboundConnectionQueueTest
Posted by rm...@apache.org.
Add type parameters in OutboundConnectionQueueTest
The use of PowerMock's Whitebox without type parameters led to compile
problems with Java 6.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ccd49506
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ccd49506
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ccd49506
Branch: refs/heads/travis_test
Commit: ccd4950689924115657ed5a764866dc63c615470
Parents: a45c14b
Author: uce <u....@fu-berlin.de>
Authored: Thu Jul 31 11:47:41 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Thu Jul 31 11:47:41 2014 +0200
----------------------------------------------------------------------
.../runtime/io/network/netty/OutboundConnectionQueueTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ccd49506/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
index 1f4d98a..e2f3a14 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
@@ -154,7 +154,7 @@ public class OutboundConnectionQueueTest {
Mockito.verify(connectionManager, Mockito.times(0)).close(Mockito.any(RemoteReceiver.class));
- boolean hasRequestedClose = Whitebox.getInternalState(queue, "hasRequestedClose");
+ Boolean hasRequestedClose = Whitebox.<Boolean>getInternalState(queue, "hasRequestedClose");
Assert.assertFalse("Close request while envelope in flight.", hasRequestedClose);
// Change writability of channel back to writable
[32/39] git commit: Create release scripts,
add support for "cdh4" build.
Posted by rm...@apache.org.
Create release scripts, add support for "cdh4" build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ecb829ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ecb829ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ecb829ed
Branch: refs/heads/travis_test
Commit: ecb829ed47544242953db8c1806338445c40713d
Parents: e4c3754
Author: Robert Metzger <rm...@apache.org>
Authored: Wed Jul 30 19:57:21 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Aug 6 11:43:05 2014 +0200
----------------------------------------------------------------------
DISCLAIMER | 11 ++
flink-addons/flink-hadoop-compatibility/pom.xml | 23 ++++
.../mapred/wrapper/HadoopDummyReporter.java | 3 +-
flink-clients/pom.xml | 6 +
flink-dist/pom.xml | 24 +++-
flink-dist/src/main/assemblies/bin.xml | 5 +-
flink-dist/src/main/assemblies/yarn.xml | 9 +-
flink-runtime/pom.xml | 25 ++++
flink-test-utils/pom.xml | 12 ++
flink-tests/pom.xml | 12 ++
pom.xml | 1 +
tools/create_release_files.sh | 134 +++++++++++++++++++
tools/how_to_release.md | 16 +++
tools/prepare_release.sh | 23 ++++
14 files changed, 291 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/DISCLAIMER
----------------------------------------------------------------------
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..48a1a19
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,11 @@
+Apache Flink is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/pom.xml b/flink-addons/flink-hadoop-compatibility/pom.xml
index 746b782..52c2917 100644
--- a/flink-addons/flink-hadoop-compatibility/pom.xml
+++ b/flink-addons/flink-hadoop-compatibility/pom.xml
@@ -78,5 +78,28 @@ under the License.
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>cdh4</id>
+ <!--
+ See flink-runtime/pom.xml for details
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
index 87a6014..ac2b4b6 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/wrapper/HadoopDummyReporter.java
@@ -62,8 +62,7 @@ public class HadoopDummyReporter implements Reporter {
public InputSplit getInputSplit() throws UnsupportedOperationException {
return null;
}
-
- @Override
+ // There should be an @Override, but some CDH4 dependency does not contain this method
public float getProgress() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 7456209..b035c19 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -95,6 +95,12 @@ under the License.
<version>2.4</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.4</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<!-- More information on this:
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 5672fde..8c46149 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -132,8 +132,9 @@ under the License.
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<executions>
+ <!-- Uber-jar -->
<execution>
- <id>single-jar</id>
+ <id>uber-jar</id>
<phase>package</phase>
<goals>
<goal>single</goal>
@@ -154,8 +155,23 @@ under the License.
</archive>
<descriptors>
<descriptor>src/main/assemblies/yarn-uberjar.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+
+ <!-- yarn bin directory -->
+ <execution>
+ <id>yarn-bin</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
<descriptor>src/main/assemblies/yarn.xml</descriptor>
</descriptors>
+ <finalName>flink-${project.version}-bin</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
@@ -321,13 +337,15 @@ under the License.
</profile>
</profiles>
+
+
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
- <id>generate-package</id>
+ <id>bin</id>
<phase>package</phase>
<goals>
<goal>single</goal>
@@ -336,6 +354,8 @@ under the License.
<descriptors>
<descriptor>src/main/assemblies/bin.xml</descriptor>
</descriptors>
+ <finalName>flink-${project.version}-bin</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/bin.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cc5850a..6ade3be 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -23,10 +23,6 @@ under the License.
<id>bin</id>
<formats>
<format>dir</format>
- <!--
- <format>tar.gz</format>
- <format>zip</format>
- -->
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
@@ -114,6 +110,7 @@ under the License.
<include>*.txt</include>
<include>LICENSE*</include>
<include>NOTICE*</include>
+ <include>DISCLAIMER*</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-dist/src/main/assemblies/yarn.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/yarn.xml b/flink-dist/src/main/assemblies/yarn.xml
index 2fa19cb..f262ab0 100644
--- a/flink-dist/src/main/assemblies/yarn.xml
+++ b/flink-dist/src/main/assemblies/yarn.xml
@@ -23,11 +23,7 @@ under the License.
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>yarn</id>
<formats>
- <format>tar.gz</format>
- <!--
- <format>tar.gz</format>
- <format>zip</format>
- -->
+ <format>dir</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
@@ -68,6 +64,9 @@ under the License.
<includes>
<include>*.txt</include>
<include>*.properties</include>
+ <include>DISCLAIMER*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 7bb49f4..7b22708 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -217,5 +217,30 @@ under the License.
</dependency>
</dependencies>
</profile>
+
+ <profile>
+ <id>cdh4</id>
+ <!--
+
+ It seems http://www.cloudera.com/content/cloudera-content/cloudera-docs/CDH4/4.2.0/CDH4-Installation-Guide/cdh4ig_topic_31.html
+ that this particular CDH4 version has already separate packages for hdfs and common.
+
+ To activate the cdh4 profile, use the -Pcdh4 flag. (You probably want -Pcdh4,vendor-repos)
+ You also HAVE TO set the -Dhadoop.cdh4.hadoop.version property (for example to: 2.0.0-cdh4.2.0)
+
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index 1a0fe1b..a9d21ed 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -66,4 +66,16 @@ under the License.
<version>4.11</version>
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>cdh4</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 950e78e..b4b11ea 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -236,4 +236,16 @@ under the License.
</plugins>
</pluginManagement>
</build>
+ <profiles>
+ <profile>
+ <id>cdh4</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.cdh4.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8058539..a22c7e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -460,6 +460,7 @@ under the License.
<exclude>**/target/**</exclude>
<!-- Documentation -->
<exclude>**/docs/**</exclude>
+ <exclude>**/tools/*.md</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
new file mode 100755
index 0000000..aa8875d
--- /dev/null
+++ b/tools/create_release_files.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Quick-and-dirty automation of making maven and binary releases. Not robust at all.
+# Publishes releases to Maven and packages/copies binary release artifacts.
+# Expects to be run in a totally empty directory.
+#
+
+#
+# NOTE: The code in this file is based on code from the Apache Spark
+# project, licensed under the Apache License v 2.0
+#
+# https://github.com/apache/spark/blob/master/dev/create-release/create-release.sh
+#
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "tools" ]] ; then
+ echo "You have to call the script from the tools/ dir"
+ exit 1
+fi
+
+GIT_USERNAME=${GIT_USERNAME:-pwendell}
+GIT_PASSWORD=${GIT_PASSWORD:-XXX}
+GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
+GPG_KEY=${GPG_KEY:-XXX}
+GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
+RELEASE_VERSION=${NEW_VERSION}
+USER_NAME=${USER_NAME:-pwendell}
+MVN=${MVN:-mvn}
+
+set -e
+
+# create source package
+
+git clone https://github.com/rmetzger/incubator-flink.git flink
+cd flink
+git checkout -b release-0.6 origin/release-0.6
+rm .gitignore
+rm .travis.yml
+rm deploysettings.xml
+rm CHANGELOG
+rm -rf .git
+cd ..
+
+
+cp -r flink flink-$RELEASE_VERSION
+tar cvzf flink-$RELEASE_VERSION.tgz flink-$RELEASE_VERSION
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --armour --output flink-$RELEASE_VERSION.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION.tgz
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md MD5 flink-$RELEASE_VERSION.tgz > \
+ flink-$RELEASE_VERSION.tgz.md5
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md SHA512 flink-$RELEASE_VERSION.tgz > \
+ flink-$RELEASE_VERSION.tgz.sha
+rm -rf flink-$RELEASE_VERSION
+
+
+make_binary_release() {
+ NAME=$1
+ FLAGS=$2
+ cp -r flink flink-$RELEASE_VERSION-bin-$NAME
+
+ cd flink-$RELEASE_VERSION-bin-$NAME
+ # make distribution
+ $MVN clean package $FLAGS -DskipTests
+ cd flink-dist/target/flink-$RELEASE_VERSION-bin/
+ tar cvzf flink-$RELEASE_VERSION-bin-$NAME.tgz flink-$RELEASE_VERSION
+ if [ -d "flink-yarn-$RELEASE_VERSION" ] ; then
+ tar cvzf flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz flink-yarn-$RELEASE_VERSION
+ fi
+ cp flink-*.tgz ../../../../
+ cd ../../../../
+ rm -rf flink-$RELEASE_VERSION
+
+ # Sign md5 and sha the tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --armour \
+ --output flink-$RELEASE_VERSION-bin-$NAME.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION-bin-$NAME.tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ MD5 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME.tgz.md5
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ SHA512 flink-$RELEASE_VERSION-bin-$NAME.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME.tgz.sha
+
+ if [ -f "flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz" ] ; then
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --armour \
+ --output flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ MD5 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.md5
+ echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY \
+ --passphrase-fd 0 --print-md \
+ SHA512 flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz > \
+ flink-$RELEASE_VERSION-bin-$NAME-yarn.tgz.sha
+ fi
+}
+
+make_binary_release "hadoop1" ""
+make_binary_release "cdh4" "-Pvendor-repos,cdh4 -Dhadoop.version=2.0.0-mr1-cdh4.2.0 -Dhadoop.cdh4.hadoop.version=2.0.0-cdh4.2.0"
+# make_binary_release "cdh5" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0+cdh5.1.0+795"
+make_binary_release "hadoop2" "-Dhadoop.profile=2"
+# make_binary_release "mapr4" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.3.0-mapr-4.0.0-FCS"
+# make_binary_release "hdp21" "-Dhadoop.profile=2 -Pvendor-repos -Dhadoop.version=2.4.0.2.1.3.0-563"
+
+
+# Copy data
+echo "Copying release tarballs"
+folder=flink-$RELEASE_VERSION
+ssh $USER_NAME@people.apache.org \
+ mkdir /home/$USER_NAME/public_html/$folder
+scp flink-* \
+ $USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$folder/
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/how_to_release.md
----------------------------------------------------------------------
diff --git a/tools/how_to_release.md b/tools/how_to_release.md
new file mode 100644
index 0000000..65cc3b7
--- /dev/null
+++ b/tools/how_to_release.md
@@ -0,0 +1,16 @@
+# How to release a new version of Apache Flink
+
+
+You need to have the following environment variables set:
+
+export OLD_VERSION=0.6-SNAPSHOT
+export NEW_VERSION=0.6
+export GPG_KEY=XX
+export GPG_PASSPHRASE=XX
+
+./prepare_release_files.sh
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ecb829ed/tools/prepare_release.sh
----------------------------------------------------------------------
diff --git a/tools/prepare_release.sh b/tools/prepare_release.sh
new file mode 100644
index 0000000..17354b9
--- /dev/null
+++ b/tools/prepare_release.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Updating version from $OLD_VERSION to $NEW_VERSION"
+
+find .. -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#' {} \;
+
[27/39] [FLINK-1023] Switch group-at-a-time function to
java.lang.Iterable (from java.util.Iterator) Iterables over transient data
throw an TraversableOnceException when iterated over again.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/recordJobs/graph/WorksetConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/graph/WorksetConnectedComponents.java b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/graph/WorksetConnectedComponents.java
index 52f14c6..07fd658 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/graph/WorksetConnectedComponents.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/graph/WorksetConnectedComponents.java
@@ -79,7 +79,7 @@ public class WorksetConnectedComponents implements Program, ProgramDescription {
}
/**
- * Minimum aggregation over (Vertex-ID, Component-ID) pairs, selecting the pair with the smallest Comonent-ID.
+ * Minimum aggregation over (Vertex-ID, Component-ID) pairs, selecting the pair with the smallest Component-ID.
*/
@Combinable
@ConstantFields(0)
[06/39] git commit: Fix License and Notice entries and headers for
all web frontend files (css, js, fonts)
Posted by rm...@apache.org.
Fix License and Notice entries and headers for all web frontend files (css, js, fonts)
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ac6fd75c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ac6fd75c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ac6fd75c
Branch: refs/heads/travis_test
Commit: ac6fd75c0e8756d62d7aeaf41269afd2930fe878
Parents: 25b4099
Author: Stephan Ewen <se...@apache.org>
Authored: Mon Jul 28 19:00:11 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon Jul 28 19:00:11 2014 +0200
----------------------------------------------------------------------
LICENSE | 22 +++-
flink-dist/src/main/flink-bin/LICENSE | 34 ++++--
.../resources/web-docs-infoserver/analyze.html | 4 +-
.../web-docs-infoserver/css/nephelefrontend.css | 18 +++
.../img/StratosphereLogo.png | Bin 12127 -> 0 bytes
.../web-docs-infoserver/js/analyzer.js | 20 +++-
.../web-docs-infoserver/js/configuration.js | 17 +++
.../web-docs-infoserver/js/googlejsapi.js | 39 -------
.../resources/web-docs-infoserver/js/helpers.js | 109 +++----------------
.../js/jobmanagerFrontend.js | 18 +++
.../web-docs-infoserver/js/taskmanager.js | 17 +++
11 files changed, 154 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 572b6db..c46c750 100644
--- a/LICENSE
+++ b/LICENSE
@@ -234,6 +234,8 @@ under the Apache License (v 2.0):
- Jetty Web Container (http://www.eclipse.org/jetty/)
- Amazon Web Services SDK for Java (http://aws.amazon.com/sdkforjava/)
- ScalaTest (http://www.scalatest.org)
+ - StartBootstrap (http://startbootstrap.com)
+ - CHAP Links Library Timeline (http://almende.github.io/chap-links-library/)
-----------------------------------------------------------------------
Eclipse Public License - v 1.0
@@ -256,10 +258,13 @@ The Apache Flink project depends on and/or bundles the following components
under the The MIT License
- Mockito (http://www.mockito.org) - Copyright (c) 2007 Mockito contributors
- - SLF4J (http://www.slf4j.org/) - Copyright (c) 2004-2013 QOS.ch
- - jQuery 1.4.2 (http://jquery.com/) - Copyright 2014 jQuery Foundation and other contributors
+ - SLF4J (http://www.slf4j.org) - Copyright (c) 2004-2013 QOS.ch
+ - jQuery 1.4.2 (http://jquery.com) - Copyright 2014 jQuery Foundation and other contributors
- jCanvas 13.11.21 (http://calebevans.me/projects/jcanvas/) - Copyright 2014 Caleb Evans
- - Flot 0.8.1 (http://www.flotcharts.org/) - Copyright (c) 2007-2013 IOLA and Ole Laursen
+ - Flot 0.8.1 (http://www.flotcharts.org) - Copyright (c) 2007-2013 IOLA and Ole Laursen
+ - Bootstrap (http://getbootstrap.com) - Copyright (c) 2011-2014 Twitter, Inc
+ - Font Awesome - Code (http://fortawesome.github.io/Font-Awesome/) - Copyright (c) 2014 Dave Gandy
+
All rights reserved.
@@ -332,6 +337,17 @@ POSSIBILITY OF SUCH DAMAGE.
-----------------------------------------------------------------------
+ The Open Font License
+-----------------------------------------------------------------------
+
+The Apache Flink project packages the following fonts under the
+Open Font License (OFT) - http://scripts.sil.org/OFL/
+
+ - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
+ -> fonts in "flink-runtime/resources/web-docs-infoserver/font-awesome/fonts"
+
+
+-----------------------------------------------------------------------
For Apache Hadoop Subcomponents
-----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-dist/src/main/flink-bin/LICENSE
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/LICENSE b/flink-dist/src/main/flink-bin/LICENSE
index aa537d7..13f25f8 100644
--- a/flink-dist/src/main/flink-bin/LICENSE
+++ b/flink-dist/src/main/flink-bin/LICENSE
@@ -224,16 +224,18 @@ under the Apache License (v 2.0):
- Apache Commons FileUpload (http://commons.apache.org/fileupload/)
- Apache Commons IO (http://commons.apache.org/io/)
- Apache Log4J (http://logging.apache.org/log4j/1.2/)
- - Apache Avro (http://avro.apache.org/)
+ - Apache Avro (http://avro.apache.org)
- Apache Hadoop (http://hadoop.apache.org)
- Apache Derby (http://db.apache.org/derby/)
- Google Guava (https://code.google.com/p/guava-libraries/)
- - Netty (http://netty.io/)
+ - Netty (http://netty.io)
- Powermock (http://www.powermock.org)
- Javassist (http://www.javassist.org)
- - Jetty Web Container (http://www.eclipse.org/jetty)
- - Amazon Web Services SDK for Java (http://aws.amazon.com/sdkforjava)
-
+ - Jetty Web Container (http://www.eclipse.org/jetty/)
+ - Amazon Web Services SDK for Java (http://aws.amazon.com/sdkforjava/)
+ - ScalaTest (http://www.scalatest.org)
+ - StartBootstrap (http://startbootstrap.com)
+ - CHAP Links Library Timeline (http://almende.github.io/chap-links-library/)
-----------------------------------------------------------------------
Eclipse Public License - v 1.0
@@ -256,10 +258,13 @@ The Apache Flink project depends on and/or bundles the following components
under the The MIT License
- Mockito (http://www.mockito.org) - Copyright (c) 2007 Mockito contributors
- - SLF4J (http://www.slf4j.org/) - Copyright (c) 2004-2013 QOS.ch
- - jQuery 1.4.2 (http://jquery.com/) - Copyright 2014 jQuery Foundation and other contributors
+ - SLF4J (http://www.slf4j.org) - Copyright (c) 2004-2013 QOS.ch
+ - jQuery 1.4.2 (http://jquery.com) - Copyright 2014 jQuery Foundation and other contributors
- jCanvas 13.11.21 (http://calebevans.me/projects/jcanvas/) - Copyright 2014 Caleb Evans
- - Flot 0.8.1 (http://www.flotcharts.org/) - Copyright (c) 2007-2013 IOLA and Ole Laursen
+ - Flot 0.8.1 (http://www.flotcharts.org) - Copyright (c) 2007-2013 IOLA and Ole Laursen
+ - Bootstrap (http://getbootstrap.com) - Copyright (c) 2011-2014 Twitter, Inc
+ - Font Awesome - Code (http://fortawesome.github.io/Font-Awesome/) - Copyright (c) 2014 Dave Gandy
+
All rights reserved.
@@ -299,7 +304,7 @@ under BSD-style licenses
- Scala Library (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
- Scala Compiler (BSD-like) - (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
- Scala Compiler Reflect (BSD-like) - (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
-
+ - ASM (BSD-like) - (http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
(Below is the 3-clause BSD license)
@@ -332,6 +337,17 @@ POSSIBILITY OF SUCH DAMAGE.
-----------------------------------------------------------------------
+ The Open Font License
+-----------------------------------------------------------------------
+
+The Apache Flink project packages the following fonts under the
+Open Font License (OFT) - http://scripts.sil.org/OFL/
+
+ - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy
+ -> fonts in "resources/web-docs-infoserver/font-awesome/fonts"
+
+
+-----------------------------------------------------------------------
For Apache Hadoop Subcomponents
-----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/analyze.html
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/analyze.html b/flink-runtime/resources/web-docs-infoserver/analyze.html
index da29c02..034e069 100755
--- a/flink-runtime/resources/web-docs-infoserver/analyze.html
+++ b/flink-runtime/resources/web-docs-infoserver/analyze.html
@@ -27,11 +27,13 @@
<script type="text/javascript" src="js/jquery.flot.min.js"></script>
<script type="text/javascript" src="js/helpers.js"></script>
<script type="text/javascript" src="js/jcanvas.min.js"></script>
- <script type="text/javascript" src="js/googlejsapi.js"></script>
<script type="text/javascript" src="js/timeline.js"></script>
<script type="text/javascript" src="js/helpers.js"></script>
<script type="text/javascript" src="js/analyzer.js"></script>
+ <!-- Scripts from Google -->
+ <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+
<!-- Load Menu -->
<script type="text/javascript">
$(document).ready(function() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/css/nephelefrontend.css
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/css/nephelefrontend.css b/flink-runtime/resources/web-docs-infoserver/css/nephelefrontend.css
index 6256b84..eba1987 100644
--- a/flink-runtime/resources/web-docs-infoserver/css/nephelefrontend.css
+++ b/flink-runtime/resources/web-docs-infoserver/css/nephelefrontend.css
@@ -1,3 +1,21 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
.opensub {
cursor: pointer
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/img/StratosphereLogo.png
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/img/StratosphereLogo.png b/flink-runtime/resources/web-docs-infoserver/img/StratosphereLogo.png
deleted file mode 100644
index 24b4630..0000000
Binary files a/flink-runtime/resources/web-docs-infoserver/img/StratosphereLogo.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/analyzer.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/analyzer.js b/flink-runtime/resources/web-docs-infoserver/js/analyzer.js
index b73fc7c..e96f420 100644
--- a/flink-runtime/resources/web-docs-infoserver/js/analyzer.js
+++ b/flink-runtime/resources/web-docs-infoserver/js/analyzer.js
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
var jsonGlobal;
var widthProgressbar = 120;
@@ -292,4 +310,4 @@ function getSelectedRow(timeline) {
}
}
return row;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/configuration.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/configuration.js b/flink-runtime/resources/web-docs-infoserver/js/configuration.js
index 31a0d3c..63bac42 100644
--- a/flink-runtime/resources/web-docs-infoserver/js/configuration.js
+++ b/flink-runtime/resources/web-docs-infoserver/js/configuration.js
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
$(document).ready(function() {
$.ajax({ url : "setupInfo?get=globalC", type : "GET", cache: false, success : function(json) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/googlejsapi.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/googlejsapi.js b/flink-runtime/resources/web-docs-infoserver/js/googlejsapi.js
deleted file mode 100644
index fd6ccef..0000000
--- a/flink-runtime/resources/web-docs-infoserver/js/googlejsapi.js
+++ /dev/null
@@ -1,39 +0,0 @@
-if(!window['googleLT_']){window['googleLT_']=(new Date()).getTime();}if (!window['google']) {
-window['google'] = {};
-}
-if (!window['google']['loader']) {
-window['google']['loader'] = {};
-google.loader.ServiceBase = 'http://www.google.com/uds';
-google.loader.GoogleApisBase = 'http://ajax.googleapis.com/ajax';
-google.loader.ApiKey = 'notsupplied';
-google.loader.KeyVerified = true;
-google.loader.LoadFailure = false;
-google.loader.Secure = false;
-google.loader.GoogleLocale = 'www.google.com';
-google.loader.ClientLocation = null;
-google.loader.AdditionalParams = '';
-(function() {var d=encodeURIComponent,g=window,h=document;function l(a,b){return a.load=b}var m="push",n="replace",q="charAt",r="indexOf",t="ServiceBase",u="name",v="getTime",w="length",x="prototype",y="setTimeout",z="loader",A="substring",B="join",C="toLowerCase";function D(a){return a in E?E[a]:E[a]=-1!=navigator.userAgent[C]()[r](a)}var E={};function F(a,b){var c=function(){};c.prototype=b[x];a.U=b[x];a.prototype=new c}
-function G(a,b,c){var e=Array[x].slice.call(arguments,2)||[];return function(){var c=e.concat(Array[x].slice.call(arguments));return a.apply(b,c)}}function H(a){a=Error(a);a.toString=function(){return this.message};return a}function I(a,b){for(var c=a.split(/\./),e=g,f=0;f<c[w]-1;f++)e[c[f]]||(e[c[f]]={}),e=e[c[f]];e[c[c[w]-1]]=b}function J(a,b,c){a[b]=c}if(!K)var K=I;if(!L)var L=J;google[z].v={};K("google.loader.callbacks",google[z].v);var M={},N={};google[z].eval={};K("google.loader.eval",google[z].eval);
-l(google,function(a,b,c){function e(a){var b=a.split(".");if(2<b[w])throw H("Module: '"+a+"' not found!");"undefined"!=typeof b[1]&&(f=b[0],c.packages=c.packages||[],c.packages[m](b[1]))}var f=a;c=c||{};if(a instanceof Array||a&&"object"==typeof a&&"function"==typeof a[B]&&"function"==typeof a.reverse)for(var k=0;k<a[w];k++)e(a[k]);else e(a);if(a=M[":"+f]){c&&!c.language&&c.locale&&(c.language=c.locale);c&&"string"==typeof c.callback&&(k=c.callback,k.match(/^[[\]A-Za-z0-9._]+$/)&&(k=g.eval(k),c.callback=
-k));if((k=c&&null!=c.callback)&&!a.s(b))throw H("Module: '"+f+"' must be loaded before DOM onLoad!");k?a.m(b,c)?g[y](c.callback,0):a.load(b,c):a.m(b,c)||a.load(b,c)}else throw H("Module: '"+f+"' not found!");});K("google.load",google.load);
-google.T=function(a,b){b?(0==O[w]&&(P(g,"load",Q),!D("msie")&&!D("safari")&&!D("konqueror")&&D("mozilla")||g.opera?g.addEventListener("DOMContentLoaded",Q,!1):D("msie")?h.write("<script defer onreadystatechange='google.loader.domReady()' src=//:>\x3c/script>"):(D("safari")||D("konqueror"))&&g[y](S,10)),O[m](a)):P(g,"load",a)};K("google.setOnLoadCallback",google.T);
-function P(a,b,c){if(a.addEventListener)a.addEventListener(b,c,!1);else if(a.attachEvent)a.attachEvent("on"+b,c);else{var e=a["on"+b];a["on"+b]=null!=e?aa([c,e]):c}}function aa(a){return function(){for(var b=0;b<a[w];b++)a[b]()}}var O=[];google[z].P=function(){var a=g.event.srcElement;"complete"==a.readyState&&(a.onreadystatechange=null,a.parentNode.removeChild(a),Q())};K("google.loader.domReady",google[z].P);var ba={loaded:!0,complete:!0};function S(){ba[h.readyState]?Q():0<O[w]&&g[y](S,10)}
-function Q(){for(var a=0;a<O[w];a++)O[a]();O.length=0}google[z].d=function(a,b,c){if(c){var e;"script"==a?(e=h.createElement("script"),e.type="text/javascript",e.src=b):"css"==a&&(e=h.createElement("link"),e.type="text/css",e.href=b,e.rel="stylesheet");(a=h.getElementsByTagName("head")[0])||(a=h.body.parentNode.appendChild(h.createElement("head")));a.appendChild(e)}else"script"==a?h.write('<script src="'+b+'" type="text/javascript">\x3c/script>'):"css"==a&&h.write('<link href="'+b+'" type="text/css" rel="stylesheet"></link>')};
-K("google.loader.writeLoadTag",google[z].d);google[z].Q=function(a){N=a};K("google.loader.rfm",google[z].Q);google[z].S=function(a){for(var b in a)"string"==typeof b&&b&&":"==b[q](0)&&!M[b]&&(M[b]=new T(b[A](1),a[b]))};K("google.loader.rpl",google[z].S);google[z].R=function(a){if((a=a.specs)&&a[w])for(var b=0;b<a[w];++b){var c=a[b];"string"==typeof c?M[":"+c]=new U(c):(c=new V(c[u],c.baseSpec,c.customSpecs),M[":"+c[u]]=c)}};K("google.loader.rm",google[z].R);google[z].loaded=function(a){M[":"+a.module].l(a)};
-K("google.loader.loaded",google[z].loaded);google[z].O=function(){return"qid="+((new Date)[v]().toString(16)+Math.floor(1E7*Math.random()).toString(16))};K("google.loader.createGuidArg_",google[z].O);I("google_exportSymbol",I);I("google_exportProperty",J);google[z].a={};K("google.loader.themes",google[z].a);google[z].a.I="//www.google.com/cse/style/look/bubblegum.css";L(google[z].a,"BUBBLEGUM",google[z].a.I);google[z].a.K="//www.google.com/cse/style/look/greensky.css";L(google[z].a,"GREENSKY",google[z].a.K);
-google[z].a.J="//www.google.com/cse/style/look/espresso.css";L(google[z].a,"ESPRESSO",google[z].a.J);google[z].a.M="//www.google.com/cse/style/look/shiny.css";L(google[z].a,"SHINY",google[z].a.M);google[z].a.L="//www.google.com/cse/style/look/minimalist.css";L(google[z].a,"MINIMALIST",google[z].a.L);google[z].a.N="//www.google.com/cse/style/look/v2/default.css";L(google[z].a,"V2_DEFAULT",google[z].a.N);function U(a){this.b=a;this.o=[];this.n={};this.e={};this.f={};this.j=!0;this.c=-1}
-U[x].g=function(a,b){var c="";void 0!=b&&(void 0!=b.language&&(c+="&hl="+d(b.language)),void 0!=b.nocss&&(c+="&output="+d("nocss="+b.nocss)),void 0!=b.nooldnames&&(c+="&nooldnames="+d(b.nooldnames)),void 0!=b.packages&&(c+="&packages="+d(b.packages)),null!=b.callback&&(c+="&async=2"),void 0!=b.style&&(c+="&style="+d(b.style)),void 0!=b.noexp&&(c+="&noexp=true"),void 0!=b.other_params&&(c+="&"+b.other_params));if(!this.j){google[this.b]&&google[this.b].JSHash&&(c+="&sig="+d(google[this.b].JSHash));
-var e=[],f;for(f in this.n)":"==f[q](0)&&e[m](f[A](1));for(f in this.e)":"==f[q](0)&&this.e[f]&&e[m](f[A](1));c+="&have="+d(e[B](","))}return google[z][t]+"/?file="+this.b+"&v="+a+google[z].AdditionalParams+c};U[x].t=function(a){var b=null;a&&(b=a.packages);var c=null;if(b)if("string"==typeof b)c=[a.packages];else if(b[w])for(c=[],a=0;a<b[w];a++)"string"==typeof b[a]&&c[m](b[a][n](/^\s*|\s*$/,"")[C]());c||(c=["default"]);b=[];for(a=0;a<c[w];a++)this.n[":"+c[a]]||b[m](c[a]);return b};
-l(U[x],function(a,b){var c=this.t(b),e=b&&null!=b.callback;if(e)var f=new W(b.callback);for(var k=[],p=c[w]-1;0<=p;p--){var s=c[p];e&&f.B(s);if(this.e[":"+s])c.splice(p,1),e&&this.f[":"+s][m](f);else k[m](s)}if(c[w]){b&&b.packages&&(b.packages=c.sort()[B](","));for(p=0;p<k[w];p++)s=k[p],this.f[":"+s]=[],e&&this.f[":"+s][m](f);if(b||null==N[":"+this.b]||null==N[":"+this.b].versions[":"+a]||google[z].AdditionalParams||!this.j)b&&b.autoloaded||google[z].d("script",this.g(a,b),e);else{c=N[":"+this.b];
-google[this.b]=google[this.b]||{};for(var R in c.properties)R&&":"==R[q](0)&&(google[this.b][R[A](1)]=c.properties[R]);google[z].d("script",google[z][t]+c.path+c.js,e);c.css&&google[z].d("css",google[z][t]+c.path+c.css,e)}this.j&&(this.j=!1,this.c=(new Date)[v](),1!=this.c%100&&(this.c=-1));for(p=0;p<k[w];p++)s=k[p],this.e[":"+s]=!0}});
-U[x].l=function(a){-1!=this.c&&(X("al_"+this.b,"jl."+((new Date)[v]()-this.c),!0),this.c=-1);this.o=this.o.concat(a.components);google[z][this.b]||(google[z][this.b]={});google[z][this.b].packages=this.o.slice(0);for(var b=0;b<a.components[w];b++){this.n[":"+a.components[b]]=!0;this.e[":"+a.components[b]]=!1;var c=this.f[":"+a.components[b]];if(c){for(var e=0;e<c[w];e++)c[e].C(a.components[b]);delete this.f[":"+a.components[b]]}}};U[x].m=function(a,b){return 0==this.t(b)[w]};U[x].s=function(){return!0};
-function W(a){this.F=a;this.q={};this.r=0}W[x].B=function(a){this.r++;this.q[":"+a]=!0};W[x].C=function(a){this.q[":"+a]&&(this.q[":"+a]=!1,this.r--,0==this.r&&g[y](this.F,0))};function V(a,b,c){this.name=a;this.D=b;this.p=c;this.u=this.h=!1;this.k=[];google[z].v[this[u]]=G(this.l,this)}F(V,U);l(V[x],function(a,b){var c=b&&null!=b.callback;c?(this.k[m](b.callback),b.callback="google.loader.callbacks."+this[u]):this.h=!0;b&&b.autoloaded||google[z].d("script",this.g(a,b),c)});V[x].m=function(a,b){return b&&null!=b.callback?this.u:this.h};V[x].l=function(){this.u=!0;for(var a=0;a<this.k[w];a++)g[y](this.k[a],0);this.k=[]};
-var Y=function(a,b){return a.string?d(a.string)+"="+d(b):a.regex?b[n](/(^.*$)/,a.regex):""};V[x].g=function(a,b){return this.G(this.w(a),a,b)};
-V[x].G=function(a,b,c){var e="";a.key&&(e+="&"+Y(a.key,google[z].ApiKey));a.version&&(e+="&"+Y(a.version,b));b=google[z].Secure&&a.ssl?a.ssl:a.uri;if(null!=c)for(var f in c)a.params[f]?e+="&"+Y(a.params[f],c[f]):"other_params"==f?e+="&"+c[f]:"base_domain"==f&&(b="http://"+c[f]+a.uri[A](a.uri[r]("/",7)));google[this[u]]={};-1==b[r]("?")&&e&&(e="?"+e[A](1));return b+e};V[x].s=function(a){return this.w(a).deferred};V[x].w=function(a){if(this.p)for(var b=0;b<this.p[w];++b){var c=this.p[b];if(RegExp(c.pattern).test(a))return c}return this.D};function T(a,b){this.b=a;this.i=b;this.h=!1}F(T,U);l(T[x],function(a,b){this.h=!0;google[z].d("script",this.g(a,b),!1)});T[x].m=function(){return this.h};T[x].l=function(){};T[x].g=function(a,b){if(!this.i.versions[":"+a]){if(this.i.aliases){var c=this.i.aliases[":"+a];c&&(a=c)}if(!this.i.versions[":"+a])throw H("Module: '"+this.b+"' with version '"+a+"' not found!");}return google[z].GoogleApisBase+"/libs/"+this.b+"/"+a+"/"+this.i.versions[":"+a][b&
&b.uncompressed?"uncompressed":"compressed"]};
-T[x].s=function(){return!1};var ca=!1,Z=[],da=(new Date)[v](),fa=function(){ca||(P(g,"unload",ea),ca=!0)},ga=function(a,b){fa();if(!(google[z].Secure||google[z].Options&&!1!==google[z].Options.csi)){for(var c=0;c<a[w];c++)a[c]=d(a[c][C]()[n](/[^a-z0-9_.]+/g,"_"));for(c=0;c<b[w];c++)b[c]=d(b[c][C]()[n](/[^a-z0-9_.]+/g,"_"));g[y](G($,null,"//gg.google.com/csi?s=uds&v=2&action="+a[B](",")+"&it="+b[B](",")),1E4)}},X=function(a,b,c){c?ga([a],[b]):(fa(),Z[m]("r"+Z[w]+"="+d(a+(b?"|"+b:""))),g[y](ea,5<Z[w]?0:15E3))},ea=function(){if(Z[w]){var a=
-google[z][t];0==a[r]("http:")&&(a=a[n](/^http:/,"https:"));$(a+"/stats?"+Z[B]("&")+"&nc="+(new Date)[v]()+"_"+((new Date)[v]()-da));Z.length=0}},$=function(a){var b=new Image,c=$.H++;$.A[c]=b;b.onload=b.onerror=function(){delete $.A[c]};b.src=a;b=null};$.A={};$.H=0;I("google.loader.recordCsiStat",ga);I("google.loader.recordStat",X);I("google.loader.createImageForLogging",$);
-
-}) ();google.loader.rm({"specs":["feeds","spreadsheets","gdata","visualization",{"name":"sharing","baseSpec":{"uri":"http://www.google.com/s2/sharing/js","ssl":null,"key":{"string":"key"},"version":{"string":"v"},"deferred":false,"params":{"language":{"string":"hl"}}}},"search","orkut","ads","elements",{"name":"books","baseSpec":{"uri":"http://books.google.com/books/api.js","ssl":"https://encrypted.google.com/books/api.js","key":{"string":"key"},"version":{"string":"v"},"deferred":true,"params":{"callback":{"string":"callback"},"language":{"string":"hl"}}}},{"name":"friendconnect","baseSpec":{"uri":"http://www.google.com/friendconnect/script/friendconnect.js","ssl":null,"key":{"string":"key"},"version":{"string":"v"},"deferred":false,"params":{}}},"identitytoolkit","ima",{"name":"maps","baseSpec":{"uri":"http://maps.google.com/maps?file\u003dgoogleapi","ssl":"https://maps-api-ssl.google.com/maps?file\u003dgoogleapi","key":{"string":"key"},"version":{"string":"v"},"deferred":true,"pa
rams":{"callback":{"regex":"callback\u003d$1\u0026async\u003d2"},"language":{"string":"hl"}}},"customSpecs":[{"uri":"http://maps.googleapis.com/maps/api/js","ssl":"https://maps.googleapis.com/maps/api/js","version":{"string":"v"},"deferred":true,"params":{"callback":{"string":"callback"},"language":{"string":"hl"}},"pattern":"^(3|3..*)$"}]},"payments","wave","annotations_v2","earth","language",{"name":"annotations","baseSpec":{"uri":"http://www.google.com/reviews/scripts/annotations_bootstrap.js","ssl":null,"key":{"string":"key"},"version":{"string":"v"},"deferred":true,"params":{"callback":{"string":"callback"},"language":{"string":"hl"},"country":{"string":"gl"}}}},"picker"]});
-google.loader.rfm({":search":{"versions":{":1":"1",":1.0":"1"},"path":"/api/search/1.0/65b21018ad4df09e3eb5a21326b72d0b/","js":"default+en.I.js","css":"default+en.css","properties":{":JSHash":"65b21018ad4df09e3eb5a21326b72d0b",":NoOldNames":false,":Version":"1.0"}},":language":{"versions":{":1":"1",":1.0":"1"},"path":"/api/language/1.0/f66252f80431973fa38526a329b0a47b/","js":"default+en.I.js","properties":{":JSHash":"f66252f80431973fa38526a329b0a47b",":Version":"1.0"}},":feeds":{"versions":{":1":"1",":1.0":"1"},"path":"/api/feeds/1.0/975cfa817ddcf5992ebfc12a1f34c97b/","js":"default+en.I.js","css":"default+en.css","properties":{":JSHash":"975cfa817ddcf5992ebfc12a1f34c97b",":Version":"1.0"}},":spreadsheets":{"versions":{":0":"1",":0.4":"1"},"path":"/api/spreadsheets/0.4/87ff7219e9f8a8164006cbf28d5e911a/","js":"default.I.js","properties":{":JSHash":"87ff7219e9f8a8164006cbf28d5e911a",":Version":"0.4"}},":ima":{"versions":{":3":"1",":3.0":"1"},"path":"/api/ima/3.0/28a914332232c9a8ac0ae8d
a68b1006e/","js":"default.I.js","properties":{":JSHash":"28a914332232c9a8ac0ae8da68b1006e",":Version":"3.0"}},":wave":{"versions":{":1":"1",":1.0":"1"},"path":"/api/wave/1.0/3b6f7573ff78da6602dda5e09c9025bf/","js":"default.I.js","properties":{":JSHash":"3b6f7573ff78da6602dda5e09c9025bf",":Version":"1.0"}},":earth":{"versions":{":1":"1",":1.0":"1"},"path":"/api/earth/1.0/109c7b2bae7fe6cc34ea875176165d81/","js":"default.I.js","properties":{":JSHash":"109c7b2bae7fe6cc34ea875176165d81",":Version":"1.0"}},":annotations":{"versions":{":1":"1",":1.0":"1"},"path":"/api/annotations/1.0/bacce7b6155a1bbadda3c05d65391b22/","js":"default+en.I.js","properties":{":JSHash":"bacce7b6155a1bbadda3c05d65391b22",":Version":"1.0"}},":picker":{"versions":{":1":"1",":1.0":"1"},"path":"/api/picker/1.0/1c635e91b9d0c082c660a42091913907/","js":"default.I.js","css":"default.css","properties":{":JSHash":"1c635e91b9d0c082c660a42091913907",":Version":"1.0"}}});
-google.loader.rpl({":scriptaculous":{"versions":{":1.8.3":{"uncompressed":"scriptaculous.js","compressed":"scriptaculous.js"},":1.9.0":{"uncompressed":"scriptaculous.js","compressed":"scriptaculous.js"},":1.8.2":{"uncompressed":"scriptaculous.js","compressed":"scriptaculous.js"},":1.8.1":{"uncompressed":"scriptaculous.js","compressed":"scriptaculous.js"}},"aliases":{":1.8":"1.8.3",":1":"1.9.0",":1.9":"1.9.0"}},":yui":{"versions":{":2.6.0":{"uncompressed":"build/yuiloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":2.9.0":{"uncompressed":"build/yuiloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":2.7.0":{"uncompressed":"build/yuiloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":2.8.0r4":{"uncompressed":"build/yuiloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":2.8.2r1":{"uncompressed":"build/yuiloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":2.8.1":{"uncompressed":"build/yu
iloader/yuiloader.js","compressed":"build/yuiloader/yuiloader-min.js"},":3.3.0":{"uncompressed":"build/yui/yui.js","compressed":"build/yui/yui-min.js"}},"aliases":{":3":"3.3.0",":2":"2.9.0",":2.7":"2.7.0",":2.8.2":"2.8.2r1",":2.6":"2.6.0",":2.9":"2.9.0",":2.8":"2.8.2r1",":2.8.0":"2.8.0r4",":3.3":"3.3.0"}},":swfobject":{"versions":{":2.1":{"uncompressed":"swfobject_src.js","compressed":"swfobject.js"},":2.2":{"uncompressed":"swfobject_src.js","compressed":"swfobject.js"}},"aliases":{":2":"2.2"}},":webfont":{"versions":{":1.0.28":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.27":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.29":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.12":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.13":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.14":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.15":{"uncompressed":"webfont_debug.
js","compressed":"webfont.js"},":1.0.10":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.11":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.2":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.1":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.0":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.6":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.19":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.5":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.18":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.4":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.17":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.3":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.16":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.9":{"uncompressed":"webfont_deb
ug.js","compressed":"webfont.js"},":1.0.21":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.22":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.25":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.26":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.23":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"},":1.0.24":{"uncompressed":"webfont_debug.js","compressed":"webfont.js"}},"aliases":{":1":"1.0.29",":1.0":"1.0.29"}},":ext-core":{"versions":{":3.1.0":{"uncompressed":"ext-core-debug.js","compressed":"ext-core.js"},":3.0.0":{"uncompressed":"ext-core-debug.js","compressed":"ext-core.js"}},"aliases":{":3":"3.1.0",":3.0":"3.0.0",":3.1":"3.1.0"}},":mootools":{"versions":{":1.3.1":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.1.1":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.3.0":{"uncompressed":"mootools.js","compressed":"mootools-yui-comp
ressed.js"},":1.3.2":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.1.2":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.2.3":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.2.4":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.2.1":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.2.2":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.2.5":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.4.0":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.4.1":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"},":1.4.2":{"uncompressed":"mootools.js","compressed":"mootools-yui-compressed.js"}},"aliases":{":1":"1.1.2",":1.11":"1.1.1",":1.4":"1.4.2",":1.3":"1.3.2",":1.2":"1.2.5",":1.1":"1.1.2"}},":jqueryui":{"versions":{":1.8.0":{"uncompressed":"jquery-ui.j
s","compressed":"jquery-ui.min.js"},":1.8.2":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.1":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.15":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.14":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.13":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.12":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.11":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.10":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.17":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.16":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.6.0":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.9":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.7":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.
8.8":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.7.2":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.5":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.7.3":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.6":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.7.0":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.7.1":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.8.4":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.5.3":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"},":1.5.2":{"uncompressed":"jquery-ui.js","compressed":"jquery-ui.min.js"}},"aliases":{":1.8":"1.8.17",":1.7":"1.7.3",":1.6":"1.6.0",":1":"1.8.17",":1.5":"1.5.3",":1.8.3":"1.8.4"}},":chrome-frame":{"versions":{":1.0.2":{"uncompressed":"CFInstall.js","compressed":"CFInstall.min.js"},":1.0.1":{"uncompressed":"CFInstall.js","compressed":"CFInstall
.min.js"},":1.0.0":{"uncompressed":"CFInstall.js","compressed":"CFInstall.min.js"}},"aliases":{":1":"1.0.2",":1.0":"1.0.2"}},":prototype":{"versions":{":1.7.0.0":{"uncompressed":"prototype.js","compressed":"prototype.js"},":1.6.0.2":{"uncompressed":"prototype.js","compressed":"prototype.js"},":1.6.1.0":{"uncompressed":"prototype.js","compressed":"prototype.js"},":1.6.0.3":{"uncompressed":"prototype.js","compressed":"prototype.js"}},"aliases":{":1.7":"1.7.0.0",":1.6.1":"1.6.1.0",":1":"1.7.0.0",":1.6":"1.6.1.0",":1.7.0":"1.7.0.0",":1.6.0":"1.6.0.3"}},":dojo":{"versions":{":1.3.1":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.3.0":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.6.1":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.1.1":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.3.2":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js
","compressed":"dojo/dojo.xd.js"},":1.6.0":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.2.3":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.7.2":{"uncompressed":"dojo/dojo.js.uncompressed.js","compressed":"dojo/dojo.js"},":1.7.0":{"uncompressed":"dojo/dojo.js.uncompressed.js","compressed":"dojo/dojo.js"},":1.7.1":{"uncompressed":"dojo/dojo.js.uncompressed.js","compressed":"dojo/dojo.js"},":1.4.3":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.5.1":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.5.0":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.2.0":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.4.0":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed":"dojo/dojo.xd.js"},":1.4.1":{"uncompressed":"dojo/dojo.xd.js.uncompressed.js","compressed"
:"dojo/dojo.xd.js"}},"aliases":{":1.7":"1.7.2",":1":"1.6.1",":1.6":"1.6.1",":1.5":"1.5.1",":1.4":"1.4.3",":1.3":"1.3.2",":1.2":"1.2.3",":1.1":"1.1.1"}},":jquery":{"versions":{":1.6.2":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.3.1":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.6.1":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.3.0":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.6.4":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.6.3":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.3.2":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.6.0":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.2.3":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.7.0":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.7.1":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.2.6":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.4.3":{"uncompr
essed":"jquery.js","compressed":"jquery.min.js"},":1.4.4":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.5.1":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.5.0":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.4.0":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.5.2":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.4.1":{"uncompressed":"jquery.js","compressed":"jquery.min.js"},":1.4.2":{"uncompressed":"jquery.js","compressed":"jquery.min.js"}},"aliases":{":1.7":"1.7.1",":1.6":"1.6.4",":1":"1.7.1",":1.5":"1.5.2",":1.4":"1.4.4",":1.3":"1.3.2",":1.2":"1.2.6"}}});
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/helpers.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/helpers.js b/flink-runtime/resources/web-docs-infoserver/js/helpers.js
index 4e2fdc8..b0e935c 100644
--- a/flink-runtime/resources/web-docs-infoserver/js/helpers.js
+++ b/flink-runtime/resources/web-docs-infoserver/js/helpers.js
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
function formattedTimeFromTimestamp(timestamp)
{
var date = new Date(timestamp);
@@ -19,95 +37,4 @@ function convertTime(time) {
if(millis&& millis> 0) out += millis+ " " + ((millis== 1)?"msec":"msecs") + " ";
return out.trim();
}
-/*
-function randomColors(total)
-{
- var i = 360 / (total - 1); // distribute the colors evenly on the hue range
- var r = []; // hold the generated colors
- for (var x=0; x<total; x++)
- {
- r.push(hsvToRgb(i * x, 100, 100)); // you can also alternate the saturation and value for even more contrast between the colors
- }
- return r;
-}
- /**
- * HSV to RGB color conversion
- *
- * H runs from 0 to 360 degrees
- * S and V run from 0 to 100
- *
- * Ported from the excellent java algorithm by Eugene Vishnevsky at:
- * http://www.cs.rit.edu/~ncs/color/t_convert.html
- */
-/*
- function hsvToRgb(h, s, v) {
- var r, g, b;
- var i;
- var f, p, q, t;
-
- // Make sure our arguments stay in-range
- h = Math.max(0, Math.min(360, h));
- s = Math.max(0, Math.min(100, s));
- v = Math.max(0, Math.min(100, v));
-
- // We accept saturation and value arguments from 0 to 100 because that's
- // how Photoshop represents those values. Internally, however, the
- // saturation and value are calculated from a range of 0 to 1. We make
- // That conversion here.
- s /= 100;
- v /= 100;
-
- if(s == 0) {
- // Achromatic (grey)
- r = g = b = v;
- return [Math.round(r * 255), Math.round(g * 255), Math.round(b * 255)];
- }
-
- h /= 60; // sector 0 to 5
- i = Math.floor(h);
- f = h - i; // factorial part of h
- p = v * (1 - s);
- q = v * (1 - s * f);
- t = v * (1 - s * (1 - f));
-
- switch(i) {
- case 0:
- r = v;
- g = t;
- b = p;
- break;
-
- case 1:
- r = q;
- g = v;
- b = p;
- break;
-
- case 2:
- r = p;
- g = v;
- b = t;
- break;
-
- case 3:
- r = p;
- g = q;
- b = v;
- break;
-
- case 4:
- r = t;
- g = p;
- b = v;
- break;
-
- default: // case 5:
- r = v;
- g = p;
- b = q;
- }
-
- return [Math.round(r * 255), Math.round(g * 255), Math.round(b * 255)];
- }
-*/
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/jobmanagerFrontend.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/jobmanagerFrontend.js b/flink-runtime/resources/web-docs-infoserver/js/jobmanagerFrontend.js
index 37a1772..68d178b 100644
--- a/flink-runtime/resources/web-docs-infoserver/js/jobmanagerFrontend.js
+++ b/flink-runtime/resources/web-docs-infoserver/js/jobmanagerFrontend.js
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
var jsonGlobal;
var widthProgressbar = 120;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ac6fd75c/flink-runtime/resources/web-docs-infoserver/js/taskmanager.js
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/js/taskmanager.js b/flink-runtime/resources/web-docs-infoserver/js/taskmanager.js
index 14f870f..2bfbed2 100644
--- a/flink-runtime/resources/web-docs-infoserver/js/taskmanager.js
+++ b/flink-runtime/resources/web-docs-infoserver/js/taskmanager.js
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
$(document).ready(function() {
pollTaskmanagers();
[38/39] git commit: Fix broken history due to misplaced google load js
Posted by rm...@apache.org.
Fix broken history due to misplaced google load js
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/3f1e220c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/3f1e220c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/3f1e220c
Branch: refs/heads/travis_test
Commit: 3f1e220cfa9a1f58ef4227e6024cf2296429f081
Parents: b63beba
Author: Robert Metzger <rm...@apache.org>
Authored: Sun Aug 3 14:39:12 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Sat Aug 9 14:36:31 2014 +0200
----------------------------------------------------------------------
flink-runtime/resources/web-docs-infoserver/analyze.html | 7 ++++---
tools/prepare_release.sh | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f1e220c/flink-runtime/resources/web-docs-infoserver/analyze.html
----------------------------------------------------------------------
diff --git a/flink-runtime/resources/web-docs-infoserver/analyze.html b/flink-runtime/resources/web-docs-infoserver/analyze.html
index 034e069..cf1b09e 100755
--- a/flink-runtime/resources/web-docs-infoserver/analyze.html
+++ b/flink-runtime/resources/web-docs-infoserver/analyze.html
@@ -19,6 +19,10 @@
<link rel="stylesheet" type="text/css" href="css/nephelefrontend.css" />
<link rel="stylesheet" type="text/css" href="css/timeline.css">
+ <!-- Scripts from Google -->
+ <!-- This should be loaded first to ensure the availability of google.load for the scripts -->
+ <script type="text/javascript" src="https://www.google.com/jsapi"></script>
+
<!-- Scripts from Bootstrap -->
<script src="js/jquery-1.10.2.js"></script>
<script src="js/bootstrap.js"></script>
@@ -31,9 +35,6 @@
<script type="text/javascript" src="js/helpers.js"></script>
<script type="text/javascript" src="js/analyzer.js"></script>
- <!-- Scripts from Google -->
- <script type="text/javascript" src="https://www.google.com/jsapi"></script>
-
<!-- Load Menu -->
<script type="text/javascript">
$(document).ready(function() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f1e220c/tools/prepare_release.sh
----------------------------------------------------------------------
diff --git a/tools/prepare_release.sh b/tools/prepare_release.sh
index 17354b9..7accbea 100644
--- a/tools/prepare_release.sh
+++ b/tools/prepare_release.sh
@@ -19,5 +19,5 @@
echo "Updating version from $OLD_VERSION to $NEW_VERSION"
-find .. -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#' {} \;
+find flink -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#' {} \;
[19/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
index 5ca1068..a07a157 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
@@ -22,8 +22,10 @@ import java.security.InvalidParameterException;
import java.util.Arrays;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.GenericJoiner;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputSemanticProperties;
import org.apache.flink.api.common.operators.Operator;
@@ -32,19 +34,22 @@ import org.apache.flink.api.common.operators.base.JoinOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration.SolutionSetPlaceHolder;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
import org.apache.flink.api.java.operators.Keys.FieldPositionKeys;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingJoinOperator;
import org.apache.flink.api.java.operators.translation.TupleKeyExtractingMapper;
+import org.apache.flink.api.java.operators.translation.WrappingFunction;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.types.TypeInformation;
//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
import org.apache.flink.api.java.tuple.*;
+import org.apache.flink.util.Collector;
//CHECKSTYLE.ON: AvoidStarImport
/**
@@ -147,12 +152,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param <I2> The type of the second input DataSet of the Join transformation.
* @param <OUT> The type of the result of the Join transformation.
*
- * @see JoinFunction
+ * @see org.apache.flink.api.java.functions.RichFlatJoinFunction
* @see DataSet
*/
public static class EquiJoin<I1, I2, OUT> extends JoinOperator<I1, I2, OUT> {
- private final JoinFunction<I1, I2, OUT> function;
+ private final FlatJoinFunction<I1, I2, OUT> function;
@SuppressWarnings("unused")
private boolean preserve1;
@@ -160,7 +165,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
private boolean preserve2;
protected EquiJoin(DataSet<I1> input1, DataSet<I2> input2,
- Keys<I1> keys1, Keys<I2> keys2, JoinFunction<I1, I2, OUT> function,
+ Keys<I1> keys1, Keys<I2> keys2, FlatJoinFunction<I1, I2, OUT> function,
TypeInformation<OUT> returnType, JoinHint hint)
{
super(input1, input2, keys1, keys2, returnType, hint);
@@ -171,14 +176,33 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
this.function = function;
- if (!(function instanceof ProjectJoinFunction)) {
+ if (!(function instanceof ProjectFlatJoinFunction)) {
extractSemanticAnnotationsFromUdf(function.getClass());
} else {
- generateProjectionProperties(((ProjectJoinFunction<?, ?, ?>) function));
+ generateProjectionProperties(((ProjectFlatJoinFunction<?, ?, ?>) function));
}
}
- public void generateProjectionProperties(ProjectJoinFunction<?, ?, ?> pjf) {
+ protected EquiJoin(DataSet<I1> input1, DataSet<I2> input2,
+ Keys<I1> keys1, Keys<I2> keys2, FlatJoinFunction<I1, I2, OUT> generatedFunction, JoinFunction<I1, I2, OUT> function,
+ TypeInformation<OUT> returnType, JoinHint hint)
+ {
+ super(input1, input2, keys1, keys2, returnType, hint);
+
+ if (function == null) {
+ throw new NullPointerException();
+ }
+
+ this.function = generatedFunction;
+
+ if (!(generatedFunction instanceof ProjectFlatJoinFunction)) {
+ extractSemanticAnnotationsFromUdf(function.getClass());
+ } else {
+ generateProjectionProperties(((ProjectFlatJoinFunction<?, ?, ?>) generatedFunction));
+ }
+ }
+
+ public void generateProjectionProperties(ProjectFlatJoinFunction<?, ?, ?> pjf) {
DualInputSemanticProperties props = SemanticPropUtil.createProjectionPropertiesDual(pjf.getFields(), pjf.getIsFromFirst());
setSemanticProperties(props);
}
@@ -238,8 +262,8 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
int[] logicalKeyPositions1 = super.keys1.computeLogicalKeyPositions();
int[] logicalKeyPositions2 = super.keys2.computeLogicalKeyPositions();
- JoinOperatorBase<I1, I2, OUT, GenericJoiner<I1, I2, OUT>> po =
- new JoinOperatorBase<I1, I2, OUT, GenericJoiner<I1, I2, OUT>>(function,
+ JoinOperatorBase<I1, I2, OUT, FlatJoinFunction<I1, I2, OUT>> po =
+ new JoinOperatorBase<I1, I2, OUT, FlatJoinFunction<I1, I2, OUT>>(function,
new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
logicalKeyPositions1, logicalKeyPositions2,
name);
@@ -298,7 +322,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
private static <I1, I2, K, OUT> PlanUnwrappingJoinOperator<I1, I2, OUT, K> translateSelectorFunctionJoin(
Keys.SelectorFunctionKeys<I1, ?> rawKeys1, Keys.SelectorFunctionKeys<I2, ?> rawKeys2,
- JoinFunction<I1, I2, OUT> function,
+ FlatJoinFunction<I1, I2, OUT> function,
TypeInformation<I1> inputType1, TypeInformation<I2> inputType2, TypeInformation<OUT> outputType, String name,
Operator<I1> input1, Operator<I2> input2)
{
@@ -313,10 +337,10 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
final KeyExtractingMapper<I1, K> extractor1 = new KeyExtractingMapper<I1, K>(keys1.getKeyExtractor());
final KeyExtractingMapper<I2, K> extractor2 = new KeyExtractingMapper<I2, K>(keys2.getKeyExtractor());
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingJoinOperator<I1, I2, OUT, K> join = new PlanUnwrappingJoinOperator<I1, I2, OUT, K>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
join.setFirstInput(keyMapper1);
@@ -333,7 +357,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
private static <I1, I2, K, OUT> PlanUnwrappingJoinOperator<I1, I2, OUT, K> translateSelectorFunctionJoinRight(
int[] logicalKeyPositions1, Keys.SelectorFunctionKeys<I2, ?> rawKeys2,
- JoinFunction<I1, I2, OUT> function,
+ FlatJoinFunction<I1, I2, OUT> function,
TypeInformation<I1> inputType1, TypeInformation<I2> inputType2, TypeInformation<OUT> outputType, String name,
Operator<I1> input1, Operator<I2> input2)
{
@@ -350,10 +374,10 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
final TupleKeyExtractingMapper<I1, K> extractor1 = new TupleKeyExtractingMapper<I1, K>(logicalKeyPositions1[0]);
final KeyExtractingMapper<I2, K> extractor2 = new KeyExtractingMapper<I2, K>(keys2.getKeyExtractor());
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingJoinOperator<I1, I2, OUT, K> join = new PlanUnwrappingJoinOperator<I1, I2, OUT, K>(function, logicalKeyPositions1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
@@ -371,7 +395,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
private static <I1, I2, K, OUT> PlanUnwrappingJoinOperator<I1, I2, OUT, K> translateSelectorFunctionJoinLeft(
Keys.SelectorFunctionKeys<I1, ?> rawKeys1, int[] logicalKeyPositions2,
- JoinFunction<I1, I2, OUT> function,
+ FlatJoinFunction<I1, I2, OUT> function,
TypeInformation<I1> inputType1, TypeInformation<I2> inputType2, TypeInformation<OUT> outputType, String name,
Operator<I1> input1, Operator<I2> input2)
{
@@ -388,10 +412,10 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
final KeyExtractingMapper<I1, K> extractor1 = new KeyExtractingMapper<I1, K>(keys1.getKeyExtractor());
final TupleKeyExtractingMapper<I2, K> extractor2 = new TupleKeyExtractingMapper<I2, K>(logicalKeyPositions2[0]);
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingJoinOperator<I1, I2, OUT, K> join = new PlanUnwrappingJoinOperator<I1, I2, OUT, K>(function, keys1, logicalKeyPositions2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
@@ -424,28 +448,73 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
Keys<I1> keys1, Keys<I2> keys2, JoinHint hint)
{
super(input1, input2, keys1, keys2,
- (JoinFunction<I1, I2, Tuple2<I1, I2>>) new DefaultJoinFunction<I1, I2>(),
+ (RichFlatJoinFunction<I1, I2, Tuple2<I1, I2>>) new DefaultFlatJoinFunction<I1, I2>(),
new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), hint);
}
/**
- * Finalizes a Join transformation by applying a {@link JoinFunction} to each pair of joined elements.<br/>
+ * Finalizes a Join transformation by applying a {@link org.apache.flink.api.java.functions.RichFlatJoinFunction} to each pair of joined elements.<br/>
* Each JoinFunction call returns exactly one element.
*
* @param function The JoinFunction that is called for each pair of joined elements.
* @return An EquiJoin that represents the joined result DataSet
*
- * @see JoinFunction
+ * @see org.apache.flink.api.java.functions.RichFlatJoinFunction
* @see org.apache.flink.api.java.operators.JoinOperator.EquiJoin
* @see DataSet
*/
- public <R> EquiJoin<I1, I2, R> with(JoinFunction<I1, I2, R> function) {
+ public <R> EquiJoin<I1, I2, R> with(FlatJoinFunction<I1, I2, R> function) {
if (function == null) {
throw new NullPointerException("Join function must not be null.");
}
+ if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
TypeInformation<R> returnType = TypeExtractor.getJoinReturnTypes(function, getInput1Type(), getInput2Type());
return new EquiJoin<I1, I2, R>(getInput1(), getInput2(), getKeys1(), getKeys2(), function, returnType, getJoinHint());
}
+
+ public <R> EquiJoin<I1, I2, R> with (JoinFunction<I1, I2, R> function) {
+ if (function == null) {
+ throw new NullPointerException("Join function must not be null.");
+ }
+ if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
+ FlatJoinFunction generatedFunction = new WrappingFlatJoinFunction<I1, I2, R>(function);
+ TypeInformation<R> returnType = TypeExtractor.getJoinReturnTypes(function, getInput1Type(), getInput2Type());
+ return new EquiJoin<I1, I2, R>(getInput1(), getInput2(), getKeys1(), getKeys2(), generatedFunction, function, returnType, getJoinHint());
+ }
+
+ private static class WrappingFlatJoinFunction<IN1, IN2, OUT> extends WrappingFunction<JoinFunction<IN1,IN2,OUT>> implements FlatJoinFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ private WrappingFlatJoinFunction(JoinFunction<IN1, IN2, OUT> wrappedFunction) {
+ super(wrappedFunction);
+ }
+
+ @Override
+ public void join(IN1 left, IN2 right, Collector<OUT> out) throws Exception {
+ out.collect (this.wrappedFunction.join(left, right));
+ }
+ }
+
+ /*
+ private static class GeneratedFlatJoinFunction<IN1, IN2, OUT> extends FlatJoinFunction<IN1, IN2, OUT> {
+
+ private Joinable<IN1,IN2,OUT> function;
+
+ private GeneratedFlatJoinFunction(Joinable<IN1, IN2, OUT> function) {
+ this.function = function;
+ }
+
+ @Override
+ public void join(IN1 first, IN2 second, Collector<OUT> out) throws Exception {
+ out.collect(function.join(first, second));
+ }
+ }
+ */
/**
* Initiates a ProjectJoin transformation and projects the first join input<br/>
@@ -530,7 +599,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
protected ProjectJoin(DataSet<I1> input1, DataSet<I2> input2, Keys<I1> keys1, Keys<I2> keys2, JoinHint hint, int[] fields, boolean[] isFromFirst, TupleTypeInfo<OUT> returnType) {
super(input1, input2, keys1, keys2,
- new ProjectJoinFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()),
+ new ProjectFlatJoinFunction<I1, I2, OUT>(fields, isFromFirst, returnType.createSerializer().createInstance()),
returnType, hint);
}
@@ -821,20 +890,20 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
// default join functions
// --------------------------------------------------------------------------------------------
- public static final class DefaultJoinFunction<T1, T2> extends JoinFunction<T1, T2, Tuple2<T1, T2>> {
+ public static final class DefaultFlatJoinFunction<T1, T2> extends RichFlatJoinFunction<T1, T2, Tuple2<T1, T2>> {
private static final long serialVersionUID = 1L;
private final Tuple2<T1, T2> outTuple = new Tuple2<T1, T2>();
@Override
- public Tuple2<T1, T2> join(T1 first, T2 second) throws Exception {
+ public void join(T1 first, T2 second, Collector<Tuple2<T1,T2>> out) throws Exception {
outTuple.f0 = first;
outTuple.f1 = second;
- return outTuple;
+ out.collect(outTuple);
}
}
- public static final class ProjectJoinFunction<T1, T2, R extends Tuple> extends JoinFunction<T1, T2, R> {
+ public static final class ProjectFlatJoinFunction<T1, T2, R extends Tuple> extends RichFlatJoinFunction<T1, T2, R> {
private static final long serialVersionUID = 1L;
@@ -851,7 +920,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param isFromFirst List of flags indicating whether the field should be copied from the first (true) or the second (false) input.
* @param outTupleInstance An instance of an output tuple.
*/
- private ProjectJoinFunction(int[] fields, boolean[] isFromFirst, R outTupleInstance) {
+ private ProjectFlatJoinFunction(int[] fields, boolean[] isFromFirst, R outTupleInstance) {
if(fields.length != isFromFirst.length) {
throw new IllegalArgumentException("Fields and isFromFirst arrays must have same length!");
@@ -869,7 +938,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
return isFromFirst;
}
- public R join(T1 in1, T2 in2) {
+ public void join(T1 in1, T2 in2, Collector<R> out) {
for(int i=0; i<fields.length; i++) {
if(isFromFirst[i]) {
if(fields[i] >= 0) {
@@ -885,27 +954,33 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
}
}
}
- return outTuple;
+ out.collect(outTuple);
}
}
- public static final class LeftSemiJoinFunction<T1, T2> extends JoinFunction<T1, T2, T1> {
+ public static final class LeftSemiFlatJoinFunction<T1, T2> extends RichFlatJoinFunction<T1, T2, T1> {
private static final long serialVersionUID = 1L;
@Override
- public T1 join(T1 left, T2 right) throws Exception {
- return left;
+ //public T1 join(T1 left, T2 right) throws Exception {
+ // return left;
+ //}
+ public void join (T1 left, T2 right, Collector<T1> out) {
+ out.collect(left);
}
}
- public static final class RightSemiJoinFunction<T1, T2> extends JoinFunction<T1, T2, T2> {
+ public static final class RightSemiFlatJoinFunction<T1, T2> extends RichFlatJoinFunction<T1, T2, T2> {
private static final long serialVersionUID = 1L;
@Override
- public T2 join(T1 left, T2 right) throws Exception {
- return right;
+ //public T2 join(T1 left, T2 right) throws Exception {
+ // return right;
+ //}
+ public void join (T1 left, T2 right, Collector<T2> out) {
+ out.collect(right);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java
index 03c6037..eccdeec 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java
@@ -18,11 +18,10 @@
package org.apache.flink.api.java.operators;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.DataSet;
@@ -34,7 +33,7 @@ import org.apache.flink.api.java.DataSet;
* @param <IN> The type of the data set consumed by the operator.
* @param <OUT> The type of the data set created by the operator.
*
- * @see MapFunction
+ * @see org.apache.flink.api.common.functions.MapFunction
*/
public class MapOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, MapOperator<IN, OUT>> {
@@ -42,6 +41,7 @@ public class MapOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, MapOpe
public MapOperator(DataSet<IN> input, MapFunction<IN, OUT> function) {
+
super(input, TypeExtractor.getMapReturnTypes(function, input.getType()));
this.function = function;
@@ -49,11 +49,11 @@ public class MapOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, MapOpe
}
@Override
- protected org.apache.flink.api.common.operators.base.MapOperatorBase<IN, OUT, GenericMap<IN, OUT>> translateToDataFlow(Operator<IN> input) {
+ protected org.apache.flink.api.common.operators.base.MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> translateToDataFlow(Operator<IN> input) {
String name = getName() != null ? getName() : function.getClass().getName();
// create operator
- MapOperatorBase<IN, OUT, GenericMap<IN, OUT>> po = new MapOperatorBase<IN, OUT, GenericMap<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
+ MapOperatorBase<IN, OUT, MapFunction<IN, OUT>> po = new MapOperatorBase<IN, OUT, MapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
// set input
po.setInput(input);
// set dop
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
index 9e94670..dd5a3bd 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/ProjectOperator.java
@@ -20,7 +20,7 @@ package org.apache.flink.api.java.operators;
import java.util.Arrays;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.functions.SemanticPropUtil;
@@ -51,7 +51,7 @@ public class ProjectOperator<IN, OUT extends Tuple>
}
@Override
- protected org.apache.flink.api.common.operators.base.MapOperatorBase<IN, OUT, GenericMap<IN,OUT>> translateToDataFlow(Operator<IN> input) {
+ protected org.apache.flink.api.common.operators.base.MapOperatorBase<IN, OUT, MapFunction<IN,OUT>> translateToDataFlow(Operator<IN> input) {
String name = getName() != null ? getName() : "Projection " + Arrays.toString(fields);
// create operator
PlanProjectOperator<IN, OUT> ppo = new PlanProjectOperator<IN, OUT>(fields, name, getInputType(), getResultType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
deleted file mode 100644
index d88d43d..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java.operators;
-
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.common.functions.GenericMap;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.UnaryOperatorInformation;
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction.Combinable;
-import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
-import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.types.TypeInformation;
-
-import org.apache.flink.api.java.DataSet;
-
-/**
- * This operator represents the application of a "reduceGroup" function on a data set, and the
- * result data set produced by the function.
- *
- * @param <IN> The type of the data set consumed by the operator.
- * @param <OUT> The type of the data set created by the operator.
- */
-public class ReduceGroupOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, ReduceGroupOperator<IN, OUT>> {
-
- private final GroupReduceFunction<IN, OUT> function;
-
- private final Grouping<IN> grouper;
-
- private boolean combinable;
-
-
- /**
- * Constructor for a non-grouped reduce (all reduce).
- *
- * @param input The input data set to the groupReduce function.
- * @param function The user-defined GroupReduce function.
- */
- public ReduceGroupOperator(DataSet<IN> input, GroupReduceFunction<IN, OUT> function) {
- super(input, TypeExtractor.getGroupReduceReturnTypes(function, input.getType()));
-
- this.function = function;
- this.grouper = null;
- checkCombinability();
- }
-
- /**
- * Constructor for a grouped reduce.
- *
- * @param input The grouped input to be processed group-wise by the groupReduce function.
- * @param function The user-defined GroupReduce function.
- */
- public ReduceGroupOperator(Grouping<IN> input, GroupReduceFunction<IN, OUT> function) {
- super(input != null ? input.getDataSet() : null, TypeExtractor.getGroupReduceReturnTypes(function, input.getDataSet().getType()));
-
- this.function = function;
- this.grouper = input;
- checkCombinability();
-
- extractSemanticAnnotationsFromUdf(function.getClass());
- }
-
- private void checkCombinability() {
- if (function instanceof GenericCombine && function.getClass().getAnnotation(Combinable.class) != null) {
- this.combinable = true;
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Properties
- // --------------------------------------------------------------------------------------------
-
- public boolean isCombinable() {
- return combinable;
- }
-
- public void setCombinable(boolean combinable) {
- // sanity check that the function is a subclass of the combine interface
- if (combinable && !(function instanceof GenericCombine)) {
- throw new IllegalArgumentException("The function does not implement the combine interface.");
- }
-
- this.combinable = combinable;
- }
-
- @Override
- protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<?, OUT, ?> translateToDataFlow(Operator<IN> input) {
-
- String name = getName() != null ? getName() : function.getClass().getName();
-
- // distinguish between grouped reduce and non-grouped reduce
- if (grouper == null) {
- // non grouped reduce
- UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
- new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, new int[0], name);
-
- po.setCombinable(combinable);
- // set input
- po.setInput(input);
- // the degree of parallelism for a non grouped reduce can only be 1
- po.setDegreeOfParallelism(1);
- return po;
- }
-
- if (grouper.getKeys() instanceof Keys.SelectorFunctionKeys) {
-
- @SuppressWarnings("unchecked")
- Keys.SelectorFunctionKeys<IN, ?> selectorKeys = (Keys.SelectorFunctionKeys<IN, ?>) grouper.getKeys();
-
- PlanUnwrappingReduceGroupOperator<IN, OUT, ?> po = translateSelectorFunctionReducer(
- selectorKeys, function, getInputType(), getResultType(), name, input, isCombinable());
-
- po.setDegreeOfParallelism(this.getParallelism());
-
- return po;
- }
- else if (grouper.getKeys() instanceof Keys.FieldPositionKeys) {
-
- int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
- UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
- new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
-
- po.setCombinable(combinable);
- po.setInput(input);
- po.setDegreeOfParallelism(this.getParallelism());
-
- // set group order
- if (grouper instanceof SortedGrouping) {
- SortedGrouping<IN> sortedGrouper = (SortedGrouping<IN>) grouper;
-
- int[] sortKeyPositions = sortedGrouper.getGroupSortKeyPositions();
- Order[] sortOrders = sortedGrouper.getGroupSortOrders();
-
- Ordering o = new Ordering();
- for(int i=0; i < sortKeyPositions.length; i++) {
- o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]);
- }
- po.setGroupOrder(o);
- }
-
- return po;
- }
- else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
-
- int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
- UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
- new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
-
- po.setCombinable(combinable);
- po.setInput(input);
- po.setDegreeOfParallelism(this.getParallelism());
-
- return po;
- }
- else {
- throw new UnsupportedOperationException("Unrecognized key type.");
- }
-
- }
-
-
- // --------------------------------------------------------------------------------------------
-
- private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
- Keys.SelectorFunctionKeys<IN, ?> rawKeys, GroupReduceFunction<IN, OUT> function,
- TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input,
- boolean combinable)
- {
- @SuppressWarnings("unchecked")
- final Keys.SelectorFunctionKeys<IN, K> keys = (Keys.SelectorFunctionKeys<IN, K>) rawKeys;
-
- TypeInformation<Tuple2<K, IN>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, IN>>(keys.getKeyType(), inputType);
-
- KeyExtractingMapper<IN, K> extractor = new KeyExtractingMapper<IN, K>(keys.getKeyExtractor());
-
- PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer = new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable);
-
- MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor");
-
- reducer.setInput(mapper);
- mapper.setInput(input);
-
- // set the mapper's parallelism to the input parallelism to make sure it is chained
- mapper.setDegreeOfParallelism(input.getDegreeOfParallelism());
-
- return reducer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
index 12e0f89..13a6c91 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
@@ -18,13 +18,12 @@
package org.apache.flink.api.java.operators;
-import org.apache.flink.api.common.functions.GenericMap;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.java.functions.ReduceFunction;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.KeyRemovingMapper;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceOperator;
@@ -40,7 +39,7 @@ import org.apache.flink.api.java.DataSet;
*
* @param <IN> The type of the data set reduced by the operator.
*
- * @see ReduceFunction
+ * @see org.apache.flink.api.common.functions.ReduceFunction
*/
public class ReduceOperator<IN> extends SingleInputUdfOperator<IN, IN, ReduceOperator<IN>> {
@@ -83,8 +82,8 @@ public class ReduceOperator<IN> extends SingleInputUdfOperator<IN, IN, ReduceOpe
if (grouper == null) {
// non grouped reduce
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getInputType());
- ReduceOperatorBase<IN, GenericReduce<IN>> po =
- new ReduceOperatorBase<IN, GenericReduce<IN>>(function, operatorInfo, new int[0], name);
+ ReduceOperatorBase<IN, ReduceFunction<IN>> po =
+ new ReduceOperatorBase<IN, ReduceFunction<IN>>(function, operatorInfo, new int[0], name);
// set input
po.setInput(input);
@@ -109,8 +108,8 @@ public class ReduceOperator<IN> extends SingleInputUdfOperator<IN, IN, ReduceOpe
// reduce with field positions
int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getInputType());
- ReduceOperatorBase<IN, GenericReduce<IN>> po =
- new ReduceOperatorBase<IN, GenericReduce<IN>>(function, operatorInfo, logicalKeyPositions, name);
+ ReduceOperatorBase<IN, ReduceFunction<IN>> po =
+ new ReduceOperatorBase<IN, ReduceFunction<IN>>(function, operatorInfo, logicalKeyPositions, name);
// set input
po.setInput(input);
@@ -139,8 +138,8 @@ public class ReduceOperator<IN> extends SingleInputUdfOperator<IN, IN, ReduceOpe
PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<T, K>(function, keys, name, inputType, typeInfoWithKey);
- MapOperatorBase<T, Tuple2<K, T>, GenericMap<T, Tuple2<K, T>>> keyExtractingMap = new MapOperatorBase<T, Tuple2<K, T>, GenericMap<T, Tuple2<K, T>>>(extractor, new UnaryOperatorInformation<T, Tuple2<K, T>>(inputType, typeInfoWithKey), "Key Extractor");
- MapOperatorBase<Tuple2<K, T>, T, GenericMap<Tuple2<K, T>, T>> keyRemovingMap = new MapOperatorBase<Tuple2<K, T>, T, GenericMap<Tuple2<K, T>, T>>(new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<Tuple2<K, T>, T>(typeInfoWithKey, inputType), "Key Extractor");
+ MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>> keyExtractingMap = new MapOperatorBase<T, Tuple2<K, T>, MapFunction<T, Tuple2<K, T>>>(extractor, new UnaryOperatorInformation<T, Tuple2<K, T>>(inputType, typeInfoWithKey), "Key Extractor");
+ MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> keyRemovingMap = new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(new KeyRemovingMapper<T, K>(), new UnaryOperatorInformation<Tuple2<K, T>, T>(typeInfoWithKey, inputType), "Key Extractor");
keyExtractingMap.setInput(input);
reducer.setInput(keyExtractingMap);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/SingleInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/SingleInputUdfOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/SingleInputUdfOperator.java
index fa2c1aa..dcdbed4 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/SingleInputUdfOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/SingleInputUdfOperator.java
@@ -35,8 +35,8 @@ import org.apache.flink.api.java.DataSet;
/**
* The <tt>SingleInputUdfOperator</tt> is the base class of all unary operators that execute
* user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have one input (such as {@link org.apache.flink.api.java.functions.MapFunction} or
- * {@link org.apache.flink.api.java.functions.ReduceFunction}).
+ * have one input (such as {@link org.apache.flink.api.java.functions.RichMapFunction} or
+ * {@link org.apache.flink.api.java.functions.RichReduceFunction}).
* <p>
* This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
* through configuration objects, and semantic properties.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
index 89c8bb2..97b2417 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
@@ -21,16 +21,18 @@ package org.apache.flink.api.java.operators;
import java.util.Arrays;
import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
/**
* SortedGrouping is an intermediate step for a transformation on a grouped and sorted DataSet.<br/>
* The following transformation can be applied on sorted groups:
* <ul>
- * <li>{@link SortedGrouping#reduceGroup(GroupReduceFunction)},</li>
+ * <li>{@link SortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)},</li>
* </ul>
*
* @param <T> The type of the elements of the sorted and grouped DataSet.
@@ -65,23 +67,27 @@ public class SortedGrouping<T> extends Grouping<T> {
/**
* Applies a GroupReduce transformation on a grouped and sorted {@link DataSet}.<br/>
- * The transformation calls a {@link GroupReduceFunction} for each group of the DataSet.
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichGroupReduceFunction} for each group of the DataSet.
* A GroupReduceFunction can iterate over all elements of a group and emit any
* number of output elements including none.
*
* @param reducer The GroupReduceFunction that is applied on each group of the DataSet.
* @return A GroupReduceOperator that represents the reduced DataSet.
*
- * @see GroupReduceFunction
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.functions.RichGroupReduceFunction
+ * @see GroupReduceOperator
* @see DataSet
*/
- public <R> ReduceGroupOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
+ public <R> GroupReduceOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- return new ReduceGroupOperator<T, R>(this, reducer);
+ if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
+ return new GroupReduceOperator<T, R>(this, reducer);
}
+
// --------------------------------------------------------------------------------------------
// Group Operations
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/TwoInputUdfOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/TwoInputUdfOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/TwoInputUdfOperator.java
index a85ca3f..f347fef 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/TwoInputUdfOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/TwoInputUdfOperator.java
@@ -35,8 +35,8 @@ import org.apache.flink.api.java.DataSet;
/**
* The <tt>TwoInputUdfOperator</tt> is the base class of all binary operators that execute
* user-defined functions (UDFs). The UDFs encapsulated by this operator are naturally UDFs that
- * have two inputs (such as {@link org.apache.flink.api.java.functions.JoinFunction} or
- * {@link org.apache.flink.api.java.functions.CoGroupFunction}).
+ * have two inputs (such as {@link org.apache.flink.api.java.functions.RichJoinFunction} or
+ * {@link org.apache.flink.api.java.functions.RichCoGroupFunction}).
* <p>
* This class encapsulates utilities for the UDFs, such as broadcast variables, parameterization
* through configuration objects, and semantic properties.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/UdfOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/UdfOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/UdfOperator.java
index 2040a27..bf33f4e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/UdfOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/UdfOperator.java
@@ -27,8 +27,8 @@ import org.apache.flink.api.java.DataSet;
/**
* This interface marks operators as operators that execute user-defined functions (UDFs), such as
- * {@link org.apache.flink.api.java.functions.MapFunction}, {@link org.apache.flink.api.java.functions.ReduceFunction},
- * or {@link org.apache.flink.api.java.functions.CoGroupFunction}.
+ * {@link org.apache.flink.api.java.functions.RichMapFunction}, {@link org.apache.flink.api.java.functions.RichReduceFunction},
+ * or {@link org.apache.flink.api.java.functions.RichCoGroupFunction}.
* The UDF operators stand in contrast to operators that execute built-in operations, like aggregations.
*/
public interface UdfOperator<O extends UdfOperator<O>> {
@@ -39,7 +39,7 @@ public interface UdfOperator<O extends UdfOperator<O>> {
/**
* Gets the configuration parameters that will be passed to the UDF's open method
- * {@link org.apache.flink.api.common.functions.AbstractFunction#open(Configuration)}.
+ * {@link org.apache.flink.api.common.functions.AbstractRichFunction#open(Configuration)}.
* The configuration is set via the {@link #withParameters(Configuration)}
* method.
*
@@ -69,7 +69,7 @@ public interface UdfOperator<O extends UdfOperator<O>> {
/**
* Sets the configuration parameters for the UDF. These are optional parameters that are passed
- * to the UDF in the {@link org.apache.flink.api.common.functions.AbstractFunction#open(Configuration)} method.
+ * to the UDF in the {@link org.apache.flink.api.common.functions.AbstractRichFunction#open(Configuration)} method.
*
* @param parameters The configuration parameters for the UDF.
* @return The operator itself, to allow chaining function calls.
@@ -83,7 +83,7 @@ public interface UdfOperator<O extends UdfOperator<O>> {
* {@link org.apache.flink.api.common.functions.RuntimeContext#getBroadcastVariable(String)}.
*
* The runtime context itself is available in all UDFs via
- * {@link org.apache.flink.api.common.functions.AbstractFunction#getRuntimeContext()}.
+ * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getRuntimeContext()}.
*
* @param data The data set to be broadcasted.
* @param name The name under which the broadcast data set retrieved.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
index 1d9d70d..9e71ba0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
@@ -18,12 +18,14 @@
package org.apache.flink.api.java.operators;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
public class UnsortedGrouping<T> extends Grouping<T> {
@@ -90,14 +92,14 @@ public class UnsortedGrouping<T> extends Grouping<T> {
/**
* Applies a Reduce transformation on a grouped {@link DataSet}.<br/>
- * For each group, the transformation consecutively calls a {@link ReduceFunction}
+ * For each group, the transformation consecutively calls a {@link org.apache.flink.api.java.functions.RichReduceFunction}
* until only a single element for each group remains.
* A ReduceFunction combines two elements into one new element of the same type.
*
* @param reducer The ReduceFunction that is applied on each group of the DataSet.
* @return A ReduceOperator that represents the reduced DataSet.
*
- * @see ReduceFunction
+ * @see org.apache.flink.api.java.functions.RichReduceFunction
* @see ReduceOperator
* @see DataSet
*/
@@ -110,24 +112,28 @@ public class UnsortedGrouping<T> extends Grouping<T> {
/**
* Applies a GroupReduce transformation on a grouped {@link DataSet}.<br/>
- * The transformation calls a {@link GroupReduceFunction} for each group of the DataSet.
+ * The transformation calls a {@link org.apache.flink.api.java.functions.RichGroupReduceFunction} for each group of the DataSet.
* A GroupReduceFunction can iterate over all elements of a group and emit any
* number of output elements including none.
*
* @param reducer The GroupReduceFunction that is applied on each group of the DataSet.
* @return A GroupReduceOperator that represents the reduced DataSet.
*
- * @see GroupReduceFunction
- * @see ReduceGroupOperator
+ * @see org.apache.flink.api.java.functions.RichGroupReduceFunction
+ * @see GroupReduceOperator
* @see DataSet
*/
- public <R> ReduceGroupOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
+ public <R> GroupReduceOperator<T, R> reduceGroup(GroupReduceFunction<T, R> reducer) {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- return new ReduceGroupOperator<T, R>(this, reducer);
+ if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
+ return new GroupReduceOperator<T, R>(this, reducer);
}
+
// --------------------------------------------------------------------------------------------
// Group Operations
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyExtractingMapper.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyExtractingMapper.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyExtractingMapper.java
index aea99e3..c7f65f0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyExtractingMapper.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyExtractingMapper.java
@@ -19,11 +19,11 @@
package org.apache.flink.api.java.operators.translation;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
-public final class KeyExtractingMapper<T, K> extends MapFunction<T, Tuple2<K, T>> {
+public final class KeyExtractingMapper<T, K> extends RichMapFunction<T, Tuple2<K, T>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyRemovingMapper.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyRemovingMapper.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyRemovingMapper.java
index 52cbcd3..a6cd837 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyRemovingMapper.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/KeyRemovingMapper.java
@@ -18,11 +18,11 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
-public final class KeyRemovingMapper<T, K> extends MapFunction<Tuple2<K, T>, T> {
+public final class KeyRemovingMapper<T, K> extends RichMapFunction<Tuple2<K, T>, T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanFilterOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanFilterOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanFilterOperator.java
index 7fb9c0f..8ac2d01 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanFilterOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanFilterOperator.java
@@ -18,22 +18,22 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.FilterOperatorBase;
-import org.apache.flink.api.java.functions.FilterFunction;
import org.apache.flink.types.TypeInformation;
import org.apache.flink.util.Collector;
-public class PlanFilterOperator<T> extends FilterOperatorBase<T, GenericFlatMap<T, T>> {
+public class PlanFilterOperator<T> extends FilterOperatorBase<T, FlatMapFunction<T, T>> {
public PlanFilterOperator(FilterFunction<T> udf, String name, TypeInformation<T> type) {
super(new FlatMapFilter<T>(udf), new UnaryOperatorInformation<T, T>(type, type), name);
}
public static final class FlatMapFilter<T> extends WrappingFunction<FilterFunction<T>>
- implements GenericFlatMap<T, T>
+ implements FlatMapFunction<T, T>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanProjectOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanProjectOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanProjectOperator.java
index 521814c..4de7311 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanProjectOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanProjectOperator.java
@@ -18,22 +18,22 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.types.TypeInformation;
-public class PlanProjectOperator<T, R extends Tuple> extends MapOperatorBase<T, R, GenericMap<T, R>> {
+public class PlanProjectOperator<T, R extends Tuple> extends MapOperatorBase<T, R, MapFunction<T, R>> {
public PlanProjectOperator(int[] fields, String name, TypeInformation<T> inType, TypeInformation<R> outType) {
super(new MapProjector<T, R>(fields, outType.createSerializer().createInstance()), new UnaryOperatorInformation<T, R>(inType, outType), name);
}
public static final class MapProjector<T, R extends Tuple>
- extends AbstractFunction
- implements GenericMap<T, R>
+ extends AbstractRichFunction
+ implements MapFunction<T, R>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
index 20bd3b0..89290f0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
@@ -20,20 +20,19 @@ package org.apache.flink.api.java.operators.translation;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.java.functions.CoGroupFunction;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.types.TypeInformation;
import org.apache.flink.util.Collector;
public class PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>
- extends CoGroupOperatorBase<Tuple2<K, I1>, Tuple2<K, I2>, OUT, GenericCoGrouper<Tuple2<K, I1>, Tuple2<K, I2>, OUT>>
+ extends CoGroupOperatorBase<Tuple2<K, I1>, Tuple2<K, I2>, OUT, CoGroupFunction<Tuple2<K, I1>, Tuple2<K, I2>, OUT>>
{
- public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
Keys.SelectorFunctionKeys<I1, K> key1, Keys.SelectorFunctionKeys<I2, K> key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -42,7 +41,7 @@ public class PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>
key1.computeLogicalKeyPositions(), key2.computeLogicalKeyPositions(), name);
}
- public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
int[] key1, Keys.SelectorFunctionKeys<I2, K> key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -51,7 +50,7 @@ public class PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>
new int[]{0}, key2.computeLogicalKeyPositions(), name);
}
- public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingCoGroupOperator(CoGroupFunction<I1, I2, OUT> udf,
Keys.SelectorFunctionKeys<I1, K> key1, int[] key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -63,7 +62,7 @@ public class PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>
// --------------------------------------------------------------------------------------------
public static final class TupleUnwrappingCoGrouper<I1, I2, OUT, K> extends WrappingFunction<CoGroupFunction<I1, I2, OUT>>
- implements GenericCoGrouper<Tuple2<K, I1>, Tuple2<K, I2>, OUT>
+ implements CoGroupFunction<Tuple2<K, I1>, Tuple2<K, I2>, OUT>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
index c121efe..73ea004 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
@@ -18,20 +18,19 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.java.functions.JoinFunction;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.types.TypeInformation;
import org.apache.flink.util.Collector;
public class PlanUnwrappingJoinOperator<I1, I2, OUT, K>
- extends JoinOperatorBase<Tuple2<K, I1>, Tuple2<K, I2>, OUT, GenericJoiner<Tuple2<K, I1>, Tuple2<K, I2>, OUT>>
+ extends JoinOperatorBase<Tuple2<K, I1>, Tuple2<K, I2>, OUT, FlatJoinFunction<Tuple2<K, I1>, Tuple2<K, I2>, OUT>>
{
- public PlanUnwrappingJoinOperator(JoinFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingJoinOperator(FlatJoinFunction<I1, I2, OUT> udf,
Keys.SelectorFunctionKeys<I1, K> key1, Keys.SelectorFunctionKeys<I2, K> key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -40,7 +39,7 @@ public class PlanUnwrappingJoinOperator<I1, I2, OUT, K>
key1.computeLogicalKeyPositions(), key2.computeLogicalKeyPositions(), name);
}
- public PlanUnwrappingJoinOperator(JoinFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingJoinOperator(FlatJoinFunction<I1, I2, OUT> udf,
int[] key1, Keys.SelectorFunctionKeys<I2, K> key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -49,7 +48,7 @@ public class PlanUnwrappingJoinOperator<I1, I2, OUT, K>
new int[]{0}, key2.computeLogicalKeyPositions(), name);
}
- public PlanUnwrappingJoinOperator(JoinFunction<I1, I2, OUT> udf,
+ public PlanUnwrappingJoinOperator(FlatJoinFunction<I1, I2, OUT> udf,
Keys.SelectorFunctionKeys<I1, K> key1, int[] key2, String name,
TypeInformation<OUT> type, TypeInformation<Tuple2<K, I1>> typeInfoWithKey1, TypeInformation<Tuple2<K, I2>> typeInfoWithKey2)
{
@@ -59,21 +58,26 @@ public class PlanUnwrappingJoinOperator<I1, I2, OUT, K>
}
public static final class TupleUnwrappingJoiner<I1, I2, OUT, K>
- extends WrappingFunction<JoinFunction<I1, I2, OUT>>
- implements GenericJoiner<Tuple2<K, I1>, Tuple2<K, I2>, OUT>
+ extends WrappingFunction<FlatJoinFunction<I1, I2, OUT>>
+ implements FlatJoinFunction<Tuple2<K, I1>, Tuple2<K, I2>, OUT>
{
private static final long serialVersionUID = 1L;
- private TupleUnwrappingJoiner(JoinFunction<I1, I2, OUT> wrapped) {
+ private TupleUnwrappingJoiner(FlatJoinFunction<I1, I2, OUT> wrapped) {
super(wrapped);
}
+ //@SuppressWarnings("unchecked")
+ //@Override
+ //public OUT join(Tuple2<K, I1> value1, Tuple2<K, I2> value2) throws Exception {
+ // return wrappedFunction.join((I1)(value1.getField(1)), (I2)(value2.getField(1)));
+ //}
+
@SuppressWarnings("unchecked")
@Override
- public void join(Tuple2<K, I1> value1, Tuple2<K, I2> value2,
- Collector<OUT> out) throws Exception {
- out.collect(wrappedFunction.join((I1)(value1.getField(1)), (I2)(value2.getField(1))));
+ public void join (Tuple2<K, I1> value1, Tuple2<K, I2> value2, Collector<OUT> collector) throws Exception {
+ wrappedFunction.join ((I1)(value1.getField(1)), (I2)(value2.getField(1)), collector);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
index 5a59664..5e80455 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
@@ -20,12 +20,11 @@ package org.apache.flink.api.java.operators.translation;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction.Combinable;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.types.TypeInformation;
@@ -35,12 +34,12 @@ import org.apache.flink.util.Collector;
* A reduce operator that takes 2-tuples (key-value pairs), and applies the group reduce operation only
* on the unwrapped values.
*/
-public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOperatorBase<Tuple2<K, IN>, OUT, GenericGroupReduce<Tuple2<K, IN>,OUT>> {
+public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOperatorBase<Tuple2<K, IN>, OUT, GroupReduceFunction<Tuple2<K, IN>,OUT>> {
public PlanUnwrappingReduceGroupOperator(GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name,
TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable)
{
- super(combinable ? new TupleUnwrappingCombinableGroupReducer<IN, OUT, K>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf),
+ super(combinable ? new TupleUnwrappingFlatCombinableGroupReducer<IN, OUT, K>((RichGroupReduceFunction) udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf),
new UnaryOperatorInformation<Tuple2<K, IN>, OUT>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name);
super.setCombinable(combinable);
@@ -48,9 +47,9 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
// --------------------------------------------------------------------------------------------
- @Combinable
- public static final class TupleUnwrappingCombinableGroupReducer<IN, OUT, K> extends WrappingFunction<GroupReduceFunction<IN, OUT>>
- implements GenericGroupReduce<Tuple2<K, IN>, OUT>, GenericCombine<Tuple2<K, IN>>
+ @RichGroupReduceFunction.Combinable
+ public static final class TupleUnwrappingFlatCombinableGroupReducer<IN, OUT, K> extends WrappingFunction<RichGroupReduceFunction<IN, OUT>>
+ implements GroupReduceFunction<Tuple2<K, IN>, OUT>, FlatCombineFunction<Tuple2<K, IN>>
{
private static final long serialVersionUID = 1L;
@@ -58,7 +57,7 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
private TupleUnwrappingIterator<IN, K> iter;
private TupleWrappingCollector<IN, K> coll;
- private TupleUnwrappingCombinableGroupReducer(GroupReduceFunction<IN, OUT> wrapped) {
+ private TupleUnwrappingFlatCombinableGroupReducer(RichGroupReduceFunction<IN, OUT> wrapped) {
super(wrapped);
this.iter = new TupleUnwrappingIterator<IN, K>();
this.coll = new TupleWrappingCollector<IN, K>(this.iter);
@@ -85,7 +84,7 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
}
public static final class TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K> extends WrappingFunction<GroupReduceFunction<IN, OUT>>
- implements GenericGroupReduce<Tuple2<K, IN>, OUT>
+ implements GroupReduceFunction<Tuple2<K, IN>, OUT>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceOperator.java
index 66aa430..4da981c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceOperator.java
@@ -18,10 +18,9 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
-import org.apache.flink.api.java.functions.ReduceFunction;
import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.types.TypeInformation;
@@ -31,7 +30,7 @@ import org.apache.flink.types.TypeInformation;
* A reduce operator that takes 2-tuples (key-value pairs), and applies the reduce operation only
* on the unwrapped values.
*/
-public class PlanUnwrappingReduceOperator<T, K> extends ReduceOperatorBase<Tuple2<K, T>, GenericReduce<Tuple2<K, T>>> {
+public class PlanUnwrappingReduceOperator<T, K> extends ReduceOperatorBase<Tuple2<K, T>, ReduceFunction<Tuple2<K, T>>> {
public PlanUnwrappingReduceOperator(ReduceFunction<T> udf, Keys.SelectorFunctionKeys<T, K> key, String name,
TypeInformation<T> type, TypeInformation<Tuple2<K, T>> typeInfoWithKey)
@@ -40,7 +39,7 @@ public class PlanUnwrappingReduceOperator<T, K> extends ReduceOperatorBase<Tuple
}
public static final class ReduceWrapper<T, K> extends WrappingFunction<ReduceFunction<T>>
- implements GenericReduce<Tuple2<K, T>>
+ implements ReduceFunction<Tuple2<K, T>>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleKeyExtractingMapper.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleKeyExtractingMapper.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleKeyExtractingMapper.java
index a915d1c..ecac775 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleKeyExtractingMapper.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleKeyExtractingMapper.java
@@ -18,12 +18,12 @@
package org.apache.flink.api.java.operators.translation;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
-public final class TupleKeyExtractingMapper<T, K> extends MapFunction<T, Tuple2<K, T>> {
+public final class TupleKeyExtractingMapper<T, K> extends RichMapFunction<T, Tuple2<K, T>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
index c98df6b..267d879 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
@@ -29,20 +29,21 @@ import org.apache.flink.api.common.accumulators.IntCounter;
import org.apache.flink.api.common.accumulators.LongCounter;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.IterationRuntimeContext;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.types.Value;
-public abstract class WrappingFunction<T extends AbstractFunction> extends AbstractFunction {
+public abstract class WrappingFunction<T extends Function> extends AbstractRichFunction {
private static final long serialVersionUID = 1L;
protected final T wrappedFunction;
-
-
+
protected WrappingFunction(T wrappedFunction) {
this.wrappedFunction = wrappedFunction;
}
@@ -50,12 +51,12 @@ public abstract class WrappingFunction<T extends AbstractFunction> extends Abstr
@Override
public void open(Configuration parameters) throws Exception {
- this.wrappedFunction.open(parameters);
+ FunctionUtils.openFunction(this.wrappedFunction, parameters);
}
@Override
public void close() throws Exception {
- this.wrappedFunction.close();
+ FunctionUtils.closeFunction(this.wrappedFunction);
}
@Override
@@ -63,13 +64,16 @@ public abstract class WrappingFunction<T extends AbstractFunction> extends Abstr
super.setRuntimeContext(t);
if (t instanceof IterationRuntimeContext) {
- this.wrappedFunction.setRuntimeContext(new WrappingIterationRuntimeContext(t));
+ FunctionUtils.setFunctionRuntimeContext(this.wrappedFunction, new WrappingIterationRuntimeContext(t));
}
else{
- this.wrappedFunction.setRuntimeContext(new WrappingRuntimeContext(t));
+ FunctionUtils.setFunctionRuntimeContext(this.wrappedFunction, new WrappingRuntimeContext(t));
}
}
-
+
+ public T getWrappedFunction () {
+ return this.wrappedFunction;
+ }
private static class WrappingRuntimeContext implements RuntimeContext {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
index 633adab..fa0ca11 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
@@ -21,15 +21,14 @@ package org.apache.flink.api.java.record.functions;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
/**
* The CoGroupFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.operators.CoGroupOperator}.
*/
-public abstract class CoGroupFunction extends AbstractFunction implements GenericCoGrouper<Record, Record, Record> {
+public abstract class CoGroupFunction extends AbstractRichFunction implements org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
index b2185a2..c4587fd 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
@@ -19,15 +19,13 @@
package org.apache.flink.api.java.record.functions;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCrosser;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
/**
* The CrossFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.operators.CrossOperator}.
*/
-public abstract class CrossFunction extends AbstractFunction implements GenericCrosser<Record, Record, Record> {
+public abstract class CrossFunction extends AbstractRichFunction implements org.apache.flink.api.common.functions.CrossFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
@@ -35,14 +33,19 @@ public abstract class CrossFunction extends AbstractFunction implements GenericC
* This method must be implemented to provide a user implementation of a cross.
* It is called for each element of the Cartesian product of both input sets.
- * @param record1 The record from the second input.
- * @param record2 The record from the second input.
- * @param out A collector that collects all output records.
+ * @param first The record from the second input.
+ * @param second The record from the second input.
+ * @return The result of the cross UDF
*
* @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
* runtime catches an exception, it aborts the task and lets the fail-over logic
* decide whether to retry the task execution.
*/
+
+ //@Override
+ //public abstract void cross(Record record1, Record record2, Collector<Record> out) throws Exception;
+
@Override
- public abstract void cross(Record record1, Record record2, Collector<Record> out) throws Exception;
+ public abstract Record cross(Record first, Record second) throws Exception;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
index 0222c63..dce24a3 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
@@ -19,8 +19,8 @@
package org.apache.flink.api.java.record.functions;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
@@ -28,7 +28,7 @@ import org.apache.flink.util.Collector;
* The JoinFunction must implementation by functions of a {@link org.apache.flink.api.java.operators.JoinOperator}.
* It resembles an equality join of both inputs on their key fields.
*/
-public abstract class JoinFunction extends AbstractFunction implements GenericJoiner<Record, Record, Record> {
+public abstract class JoinFunction extends AbstractRichFunction implements FlatJoinFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
@@ -38,7 +38,7 @@ public abstract class JoinFunction extends AbstractFunction implements GenericJo
*
* @param value1 The record that comes from the first input.
* @param value2 The record that comes from the second input.
- * @param out A collector that collects all output pairs.
+ * @return The result of the join UDF as record
*
* @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
* runtime catches an exception, it aborts the combine task and lets the fail-over logic
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
index 88b6282..99c945d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.java.record.functions;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
@@ -28,7 +28,7 @@ import org.apache.flink.util.Collector;
* The MapFunction must be extended to provide a mapper implementation
* By definition, the mapper is called for each individual input record.
*/
-public abstract class MapFunction extends AbstractFunction implements GenericCollectorMap<Record, Record> {
+public abstract class MapFunction extends AbstractRichFunction implements GenericCollectorMap<Record, Record> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
index 4b1dbb3..073b11a 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
@@ -21,9 +21,9 @@ package org.apache.flink.api.java.record.functions;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
@@ -31,7 +31,7 @@ import org.apache.flink.util.Collector;
* The ReduceFunction must be extended to provide a reducer implementation, as invoked by a
* {@link org.apache.flink.api.java.operators.ReduceOperator}.
*/
-public abstract class ReduceFunction extends AbstractFunction implements GenericGroupReduce<Record, Record>, GenericCombine<Record> {
+public abstract class ReduceFunction extends AbstractRichFunction implements GroupReduceFunction<Record, Record>, FlatCombineFunction<Record> {
private static final long serialVersionUID = 1L;
[07/39] git commit: [FLINK-386] Fix and clean up dependencies
Posted by rm...@apache.org.
[FLINK-386] Fix and clean up dependencies
- Upgraded to junit 4.11 and fixed code that used deprecated
- Changed scala dependency from 2.10.3 to 2.10.4 to avoid conflicts with scalatest dependency
- Upgraded log4j and commons-logging dependency
- Removed superfluous dependency on commons-codec
- Upgraded dependency versions in the base module
- Ensured consistency of avro versions across submodules
- Upgraded AWS dependency to version 1.8.1
- Unified configuration of jetty dependency
This closes #80.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/bef5e548
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/bef5e548
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/bef5e548
Branch: refs/heads/travis_test
Commit: bef5e5487b3192e5ef2169fdbbd459610b762f76
Parents: ac6fd75
Author: Daniel Warneke <wa...@apache.org>
Authored: Sun Jul 27 14:30:38 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Wed Jul 30 00:51:57 2014 +0200
----------------------------------------------------------------------
flink-addons/flink-avro/pom.xml | 4 +-
.../io/avro/AvroRecordInputFormatTest.java | 2 +-
.../api/java/io/jdbc/JDBCInputFormatTest.java | 2 +-
.../api/java/io/jdbc/JDBCOutputFormatTest.java | 2 +-
.../record/io/jdbc/JDBCInputFormatTest.java | 2 +-
.../record/io/jdbc/JDBCOutputFormatTest.java | 2 +-
flink-clients/pom.xml | 10 ++--
.../apache/flink/compiler/DOPChangeTest.java | 2 +-
.../flink/configuration/Configuration.java | 9 ++--
.../api/common/io/FileOutputFormatTest.java | 2 +-
.../common/typeutils/SerializerTestBase.java | 2 +-
.../SimpleDataDistributionTest.java | 2 +-
.../flink/types/CollectionsDataTypeTest.java | 2 +-
.../apache/flink/types/NormalizableKeyTest.java | 2 +-
.../flink/types/PrimitiveDataTypeTest.java | 10 ++--
flink-java/pom.xml | 4 +-
.../java/functions/SemanticPropUtilTest.java | 2 +-
.../java/operator/AggregateOperatorTest.java | 2 +-
.../api/java/operator/CoGroupOperatorTest.java | 2 +-
.../api/java/operator/CrossOperatorTest.java | 2 +-
.../api/java/operator/DistinctOperatorTest.java | 2 +-
.../flink/api/java/operator/GroupingTest.java | 2 +-
.../api/java/operator/JoinOperatorTest.java | 2 +-
.../java/operator/ProjectionOperatorTest.java | 2 +-
.../api/java/record/io/CsvInputFormatTest.java | 2 +-
.../api/java/record/io/CsvOutputFormatTest.java | 2 +-
...ternalProcessFixedLengthInputFormatTest.java | 2 +-
.../io/ExternalProcessInputFormatTest.java | 2 +-
.../apache/flink/api/java/tuple/Tuple2Test.java | 2 +-
.../api/java/typeutils/TypeInfoParserTest.java | 2 +-
flink-runtime/pom.xml | 8 +--
.../FileCacheDeleteValidationTest.java | 2 +-
.../instance/DefaultInstanceManagerTest.java | 1 -
.../runtime/instance/HostInClusterTest.java | 1 -
.../local/LocalInstanceManagerTest.java | 2 +-
.../flink/runtime/io/disk/ChannelViewsTest.java | 2 +-
.../runtime/io/disk/SpillingBufferTest.java | 2 +-
.../io/disk/iomanager/IOManagerITCase.java | 2 +-
.../IOManagerPerformanceBenchmark.java | 2 +-
.../io/disk/iomanager/IOManagerTest.java | 2 +-
.../netty/InboundEnvelopeDecoderTest.java | 3 +-
.../netty/NettyConnectionManagerTest.java | 2 +-
.../netty/OutboundEnvelopeEncoderTest.java | 3 +-
.../DataInputOutputSerializerTest.java | 2 +-
.../SpanningRecordSerializationTest.java | 2 +-
.../SpanningRecordSerializerTest.java | 2 +-
.../memory/DefaultMemoryManagerTest.java | 2 +-
.../flink/runtime/memory/MemorySegmentTest.java | 2 +-
.../operators/CoGroupTaskExternalITCase.java | 2 +-
.../runtime/operators/CoGroupTaskTest.java | 2 +-
.../operators/CombineTaskExternalITCase.java | 2 +-
.../runtime/operators/CombineTaskTest.java | 2 +-
.../operators/CrossTaskExternalITCase.java | 2 +-
.../flink/runtime/operators/CrossTaskTest.java | 2 +-
.../runtime/operators/DataSinkTaskTest.java | 2 +-
.../runtime/operators/DataSourceTaskTest.java | 2 +-
.../operators/MatchTaskExternalITCase.java | 2 +-
.../operators/ReduceTaskExternalITCase.java | 2 +-
.../flink/runtime/operators/ReduceTaskTest.java | 2 +-
.../resettable/BlockResettableIteratorTest.java | 2 +-
...lockResettableMutableObjectIteratorTest.java | 2 +-
.../SpillingResettableIteratorTest.java | 2 +-
...lingResettableMutableObjectIteratorTest.java | 2 +-
.../sort/AsynchonousPartialSorterITCase.java | 2 +-
.../CombiningUnilateralSortMergerITCase.java | 2 +-
.../operators/sort/ExternalSortITCase.java | 2 +-
.../operators/testutils/DriverTestBase.java | 2 +-
.../operators/testutils/TaskCancelThread.java | 2 +-
.../operators/util/OutputEmitterTest.java | 2 +-
.../operators/util/RecordOutputEmitterTest.java | 2 +-
flink-scala/pom.xml | 6 +--
flink-test-utils/pom.xml | 2 +-
flink-tests/pom.xml | 2 +-
.../test/cancelling/CancellingTestBase.java | 6 +--
.../BulkIterationWithAllReducerITCase.java | 2 +-
.../aggregators/AggregatorsITCase.java | 2 +-
...nentsWithParametrizableAggregatorITCase.java | 2 +-
.../flink/test/javaApiOperators/MapITCase.java | 2 +-
.../apache/flink/test/util/FailingTestBase.java | 2 +-
pom.xml | 51 ++++++++++++++------
80 files changed, 134 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-avro/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/pom.xml b/flink-addons/flink-avro/pom.xml
index 1ffa8d0..36ad445 100644
--- a/flink-addons/flink-avro/pom.xml
+++ b/flink-addons/flink-avro/pom.xml
@@ -51,13 +51,13 @@ under the License.
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
- <version>1.7.5</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
- <version>1.7.5</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/record/io/avro/AvroRecordInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/record/io/avro/AvroRecordInputFormatTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/record/io/avro/AvroRecordInputFormatTest.java
index 2387fd6..64fae94 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/record/io/avro/AvroRecordInputFormatTest.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/record/io/avro/AvroRecordInputFormatTest.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
index 5816fa8..41e9c5a 100644
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
+++ b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCInputFormatTest.java
@@ -24,7 +24,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.io.jdbc.JDBCInputFormat;
import org.apache.flink.api.java.tuple.Tuple2;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
index c1c899e..a101a74 100644
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
+++ b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
@@ -24,7 +24,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.io.jdbc.JDBCInputFormat;
import org.apache.flink.api.java.io.jdbc.JDBCOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
index 20463f2..295210b 100644
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
+++ b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCInputFormatTest.java
@@ -23,7 +23,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
import org.apache.flink.configuration.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
index 10ca85d..994eaf5 100644
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
+++ b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/record/io/jdbc/JDBCOutputFormatTest.java
@@ -24,7 +24,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.record.io.jdbc.JDBCInputFormat;
import org.apache.flink.api.java.record.io.jdbc.JDBCOutputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-clients/pom.xml
----------------------------------------------------------------------
diff --git a/flink-clients/pom.xml b/flink-clients/pom.xml
index 29ce3e6..7456209 100644
--- a/flink-clients/pom.xml
+++ b/flink-clients/pom.xml
@@ -64,25 +64,25 @@ under the License.
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
- <version>1.2.2</version>
+ <version>1.3.1</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -92,7 +92,7 @@ under the License.
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.1</version>
+ <version>2.4</version>
<scope>compile</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-compiler/src/test/java/org/apache/flink/compiler/DOPChangeTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/DOPChangeTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/DOPChangeTest.java
index ce14010..db1e091 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/DOPChangeTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/DOPChangeTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.compiler;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.record.operators.FileDataSink;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
index 10e6582..d7415b8 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/Configuration.java
@@ -26,12 +26,13 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.io.StringRecord;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import com.google.common.io.BaseEncoding;
+
/**
* Lightweight configuration object which can store key/value pairs. Configuration objects
* can be extracted from or integrated into the {@link GlobalConfiguration} object. They can
@@ -337,11 +338,11 @@ public class Configuration implements IOReadableWritable, java.io.Serializable {
* @return the (default) value associated with the given key.
*/
public byte[] getBytes(String key, byte[] defaultValue) {
- String encoded = getStringInternal(key);
+ final String encoded = getStringInternal(key);
if (encoded == null) {
return defaultValue;
} else {
- return Base64.decodeBase64(encoded.getBytes());
+ return BaseEncoding.base64().decode(encoded);
}
}
@@ -354,7 +355,7 @@ public class Configuration implements IOReadableWritable, java.io.Serializable {
* The bytes to be added.
*/
public void setBytes(String key, byte[] bytes) {
- String encoded = new String(Base64.encodeBase64(bytes));
+ final String encoded = BaseEncoding.base64().encode(bytes);
setStringInternal(key, encoded);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/api/common/io/FileOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/FileOutputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/FileOutputFormatTest.java
index ae5a429..fd09c07 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/FileOutputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/FileOutputFormatTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.api.common.io;
import java.io.File;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.io.FileOutputFormat;
import org.apache.flink.api.common.io.FileOutputFormat.OutputDirectoryMode;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index 7d17233..0f611ed 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -31,7 +31,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/api/distributions/SimpleDataDistributionTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/distributions/SimpleDataDistributionTest.java b/flink-core/src/test/java/org/apache/flink/api/distributions/SimpleDataDistributionTest.java
index b1fd4ba..457a075 100644
--- a/flink-core/src/test/java/org/apache/flink/api/distributions/SimpleDataDistributionTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/distributions/SimpleDataDistributionTest.java
@@ -25,7 +25,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.distributions.SimpleDistribution;
import org.apache.flink.core.memory.InputViewDataInputStreamWrapper;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/types/CollectionsDataTypeTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/types/CollectionsDataTypeTest.java b/flink-core/src/test/java/org/apache/flink/types/CollectionsDataTypeTest.java
index d77eeec..cceb8fd 100644
--- a/flink-core/src/test/java/org/apache/flink/types/CollectionsDataTypeTest.java
+++ b/flink-core/src/test/java/org/apache/flink/types/CollectionsDataTypeTest.java
@@ -28,7 +28,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Map.Entry;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.InputViewDataInputStreamWrapper;
import org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/types/NormalizableKeyTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/types/NormalizableKeyTest.java b/flink-core/src/test/java/org/apache/flink/types/NormalizableKeyTest.java
index 748eef7..c1c70e4 100644
--- a/flink-core/src/test/java/org/apache/flink/types/NormalizableKeyTest.java
+++ b/flink-core/src/test/java/org/apache/flink/types/NormalizableKeyTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.types;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.types.CharValue;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-core/src/test/java/org/apache/flink/types/PrimitiveDataTypeTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/types/PrimitiveDataTypeTest.java b/flink-core/src/test/java/org/apache/flink/types/PrimitiveDataTypeTest.java
index 64f292f..492eeb8 100644
--- a/flink-core/src/test/java/org/apache/flink/types/PrimitiveDataTypeTest.java
+++ b/flink-core/src/test/java/org/apache/flink/types/PrimitiveDataTypeTest.java
@@ -25,7 +25,7 @@ import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.InputViewDataInputStreamWrapper;
import org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper;
@@ -93,7 +93,7 @@ public class PrimitiveDataTypeTest {
public void testDoubleValue() {
DoubleValue double0 = new DoubleValue(10.2);
// test value retrieval
- Assert.assertEquals(10.2, double0.getValue());
+ Assert.assertEquals(10.2, double0.getValue(), 0.0001);
// test value comparison
DoubleValue double1 = new DoubleValue(10.2);
DoubleValue double2 = new DoubleValue(-10.5);
@@ -114,11 +114,11 @@ public class PrimitiveDataTypeTest {
double2n.read(new InputViewDataInputStreamWrapper(mIn));
double3n.read(new InputViewDataInputStreamWrapper(mIn));
Assert.assertEquals(double0.compareTo(double1n), 0);
- Assert.assertEquals(double0.getValue(), double1n.getValue());
+ Assert.assertEquals(double0.getValue(), double1n.getValue(), 0.0001);
Assert.assertEquals(double2.compareTo(double2n), 0);
- Assert.assertEquals(double2.getValue(), double2n.getValue());
+ Assert.assertEquals(double2.getValue(), double2n.getValue(), 0.0001);
Assert.assertEquals(double3.compareTo(double3n), 0);
- Assert.assertEquals(double3.getValue(), double3n.getValue());
+ Assert.assertEquals(double3.getValue(), double3n.getValue(), 0.0001);
} catch (Exception e) {
Assert.assertTrue(false);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java/pom.xml b/flink-java/pom.xml
index 2030943..06ea151 100644
--- a/flink-java/pom.xml
+++ b/flink-java/pom.xml
@@ -50,12 +50,12 @@ under the License.
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
- <version>1.7.5</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
- <version>2.23.0</version>
+ <version>2.24.0</version>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
index 631159b..d7747a7 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropUtilTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.java.functions;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.operators.DualInputSemanticProperties;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/AggregateOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/AggregateOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/AggregateOperatorTest.java
index 32c8fb2..ad2d314 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/AggregateOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/AggregateOperatorTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.java.operator;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.java.aggregation.Aggregations;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
index f7e03de..4bc7ac2 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.java.functions.KeySelector;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/CrossOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CrossOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CrossOperatorTest.java
index 59b9a03..9c88b2e 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CrossOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CrossOperatorTest.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/DistinctOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/DistinctOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/DistinctOperatorTest.java
index 3e459e8..7ef110a 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/DistinctOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/DistinctOperatorTest.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.java.functions.KeySelector;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
index 6615d3e..0938bd9 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.operators.Order;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
index a3b6c3b..8b88288 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.java.functions.KeySelector;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/operator/ProjectionOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/ProjectionOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/ProjectionOperatorTest.java
index 11e0a37..aacfb6b 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/ProjectionOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/ProjectionOperatorTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.java.operator;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvInputFormatTest.java
index dc28062..65652b8 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvInputFormatTest.java
@@ -31,7 +31,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.record.io.CsvInputFormat;
import org.apache.flink.configuration.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvOutputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvOutputFormatTest.java
index 3bf1e31..a07127e 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvOutputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/io/CsvOutputFormatTest.java
@@ -28,7 +28,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.record.io.CsvOutputFormat;
import org.apache.flink.configuration.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessFixedLengthInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessFixedLengthInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessFixedLengthInputFormatTest.java
index 8b64824..3518927 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessFixedLengthInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessFixedLengthInputFormatTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.java.record.io;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.api.java.record.io.ExternalProcessFixedLengthInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessInputFormatTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessInputFormatTest.java
index cb7a3de..1b86ecb 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessInputFormatTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/io/ExternalProcessInputFormatTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.java.record.io;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.api.java.record.io.ExternalProcessInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/tuple/Tuple2Test.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/tuple/Tuple2Test.java b/flink-java/src/test/java/org/apache/flink/api/java/tuple/Tuple2Test.java
index d4fd634..7379a88 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/tuple/Tuple2Test.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/tuple/Tuple2Test.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.java.tuple;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.tuple.Tuple2;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-java/src/test/java/org/apache/flink/api/java/typeutils/TypeInfoParserTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/TypeInfoParserTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/TypeInfoParserTest.java
index b77fc21..0e06226 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/TypeInfoParserTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/TypeInfoParserTest.java
@@ -22,7 +22,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.typeutils.BasicArrayTypeInfo;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index a1cb6e9..487beca 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -51,25 +51,25 @@ under the License.
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
- <version>8.0.0.M1</version>
+ <!-- version is derived from base module -->
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
- <version>1.2.1</version>
+ <version>1.8.1</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
index 90534fb..8e9ad2f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.filecache;
import java.io.File;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
import org.apache.flink.core.fs.local.LocalFileSystem;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/instance/DefaultInstanceManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/DefaultInstanceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/DefaultInstanceManagerTest.java
index 7beb998..6c0a7fa 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/DefaultInstanceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/DefaultInstanceManagerTest.java
@@ -31,7 +31,6 @@ import java.util.Set;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
-import org.apache.flink.runtime.instance.*;
import org.apache.flink.runtime.jobgraph.JobID;
import org.apache.flink.util.LogUtils;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/instance/HostInClusterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/HostInClusterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/HostInClusterTest.java
index f4b5c14..280b01b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/HostInClusterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/HostInClusterTest.java
@@ -30,7 +30,6 @@ import java.net.UnknownHostException;
import java.util.*;
import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.runtime.instance.*;
import org.apache.flink.runtime.jobgraph.JobID;
import org.apache.flink.runtime.topology.NetworkTopology;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/instance/local/LocalInstanceManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/local/LocalInstanceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/local/LocalInstanceManagerTest.java
index 41cb40f..9c712f8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/local/LocalInstanceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/local/LocalInstanceManagerTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.instance.local;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/ChannelViewsTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/ChannelViewsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/ChannelViewsTest.java
index c928aa8..c85e508 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/ChannelViewsTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/ChannelViewsTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.io.disk;
import java.io.EOFException;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.disk.iomanager.BlockChannelReader;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/SpillingBufferTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/SpillingBufferTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/SpillingBufferTest.java
index f4784a8..08c97d6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/SpillingBufferTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/SpillingBufferTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.io.disk;
import java.io.EOFException;
import java.util.ArrayList;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.MemorySegment;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerITCase.java
index ef4ba1b..1fab785 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerITCase.java
@@ -25,7 +25,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Random;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerPerformanceBenchmark.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerPerformanceBenchmark.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerPerformanceBenchmark.java
index 8ae8b5c..9e79311 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerPerformanceBenchmark.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerPerformanceBenchmark.java
@@ -32,7 +32,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerTest.java
index 078f05d..4b2aaee 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/disk/iomanager/IOManagerTest.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.disk.iomanager.BlockChannelAccess;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/InboundEnvelopeDecoderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/InboundEnvelopeDecoderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/InboundEnvelopeDecoderTest.java
index 6a8e2fa..54c0676 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/InboundEnvelopeDecoderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/InboundEnvelopeDecoderTest.java
@@ -23,7 +23,8 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.CompositeByteBuf;
import io.netty.channel.embedded.EmbeddedChannel;
-import junit.framework.Assert;
+
+import org.junit.Assert;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
index 6893ce9..dfb654d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.io.network.netty;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.runtime.io.network.ChannelManager;
import org.apache.flink.runtime.io.network.Envelope;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
index 7ce02ea..7ddf4c8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
@@ -21,7 +21,8 @@ package org.apache.flink.runtime.io.network.netty;
import io.netty.buffer.ByteBuf;
import io.netty.channel.embedded.EmbeddedChannel;
-import junit.framework.Assert;
+
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.network.Buffer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/DataInputOutputSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/DataInputOutputSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/DataInputOutputSerializerTest.java
index 2a784f6..b1a967c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/DataInputOutputSerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/DataInputOutputSerializerTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.io.network.serialization;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.network.serialization.DataInputDeserializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializationTest.java
index 8bcd7c4..1ea283d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializationTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.io.network.serialization;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.network.Buffer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializerTest.java
index f044ce5..1e63ef8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/serialization/SpanningRecordSerializerTest.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.io.network.serialization;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/memory/DefaultMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/memory/DefaultMemoryManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/memory/DefaultMemoryManagerTest.java
index 6918dc9..7565093 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/memory/DefaultMemoryManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/memory/DefaultMemoryManagerTest.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/memory/MemorySegmentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/memory/MemorySegmentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/memory/MemorySegmentTest.java
index 94f36ff..7a38603 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/memory/MemorySegmentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/memory/MemorySegmentTest.java
@@ -26,7 +26,7 @@ import static org.junit.Assert.fail;
import java.util.Random;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
index e8024f3..46717be 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import java.util.Iterator;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericCoGrouper;
import org.apache.flink.api.java.record.functions.CoGroupFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
index a453cf6..a99ffc8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericCoGrouper;
import org.apache.flink.api.java.record.functions.CoGroupFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
index 9091809..694ae48 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators;
import java.util.ArrayList;
import java.util.HashMap;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericGroupReduce;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
index 27fc11b..f42c443 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericGroupReduce;
import org.apache.flink.api.java.record.functions.ReduceFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
index d663d90..79d47fe 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericCrosser;
import org.apache.flink.runtime.operators.CrossDriver;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
index 0e95ebb..01015d3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import java.util.concurrent.atomic.AtomicBoolean;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericCrosser;
import org.apache.flink.api.java.record.functions.CrossFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
index e155723..07e7dcd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSinkTaskTest.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSourceTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSourceTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSourceTaskTest.java
index 9a589db..7cc3356 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSourceTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/DataSourceTaskTest.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.record.io.DelimitedInputFormat;
import org.apache.flink.core.fs.Path;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
index 97dc869..2115bc8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.GenericJoiner;
import org.apache.flink.api.java.record.functions.JoinFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
index afcbf97..bf51f41 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
index 6d8ec94..0d4e370 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableIteratorTest.java
index 4392516..96b5e09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableIteratorTest.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableMutableObjectIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableMutableObjectIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableMutableObjectIteratorTest.java
index 1ffdda4..c2040b2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableMutableObjectIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/BlockResettableMutableObjectIteratorTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.resettable;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableIteratorTest.java
index 6c45b37..f27cc73 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableIteratorTest.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.NoSuchElementException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableMutableObjectIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableMutableObjectIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableMutableObjectIteratorTest.java
index ed2db36..771b853 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableMutableObjectIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/resettable/SpillingResettableMutableObjectIteratorTest.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.resettable;
import java.util.ArrayList;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/AsynchonousPartialSorterITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/AsynchonousPartialSorterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/AsynchonousPartialSorterITCase.java
index ddb9b3d..90fe59f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/AsynchonousPartialSorterITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/AsynchonousPartialSorterITCase.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.sort;
import java.io.IOException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
index 5ddaa9f..7020f89 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
@@ -25,7 +25,7 @@ import java.util.Hashtable;
import java.util.Iterator;
import java.util.NoSuchElementException;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/ExternalSortITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/ExternalSortITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/ExternalSortITCase.java
index 427c12b..1ac2c04 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/ExternalSortITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/ExternalSortITCase.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.sort;
import java.util.Comparator;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
index c1a2fd9..9c177a6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.testutils;
import java.util.ArrayList;
import java.util.List;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.typeutils.TypeComparator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskCancelThread.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskCancelThread.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskCancelThread.java
index 49d1068..8fef47f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskCancelThread.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskCancelThread.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.testutils;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import junit.framework.Assert;
+import org.junit.Assert;
public class TaskCancelThread extends Thread
{
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
index 690ccbb..94414c9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
@@ -25,9 +25,9 @@ import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
-import junit.framework.Assert;
import junit.framework.TestCase;
+import org.junit.Assert;
import org.apache.commons.lang.NotImplementedException;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.base.IntSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/RecordOutputEmitterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/RecordOutputEmitterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/RecordOutputEmitterTest.java
index 7d71e98..7b91510 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/RecordOutputEmitterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/RecordOutputEmitterTest.java
@@ -26,7 +26,6 @@ import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.Random;
-import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.flink.api.common.distributions.DataDistribution;
@@ -44,6 +43,7 @@ import org.apache.flink.types.KeyFieldOutOfBoundsException;
import org.apache.flink.types.NullKeyFieldException;
import org.apache.flink.types.Record;
import org.apache.flink.types.StringValue;
+import org.junit.Assert;
import org.junit.Test;
public class RecordOutputEmitterTest extends TestCase {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 5b2cc0b..e9df0aa 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -54,19 +54,19 @@ under the License.
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
- <version>2.10.3</version>
+ <version>2.10.4</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>2.10.3</version>
+ <version>2.10.4</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
- <version>2.10.3</version>
+ <version>2.10.4</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/flink-test-utils/pom.xml b/flink-test-utils/pom.xml
index bed182a..1a0fe1b 100644
--- a/flink-test-utils/pom.xml
+++ b/flink-test-utils/pom.xml
@@ -63,7 +63,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.7</version>
+ <version>4.11</version>
</dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 1a94baf..e6ce7c1 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -64,7 +64,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.7</version>
+ <version>4.11</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
index bef0f71..ccd5804 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/cancelling/CancellingTestBase.java
@@ -19,11 +19,9 @@
package org.apache.flink.test.cancelling;
-import static junit.framework.Assert.fail;
-
import java.util.Iterator;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -229,7 +227,7 @@ public abstract class CancellingTestBase {
} catch (Exception e) {
LOG.error(e);
- fail(StringUtils.stringifyException(e));
+ Assert.fail(StringUtils.stringifyException(e));
return;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
index e17c089..857f321 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
@@ -29,7 +29,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
index 38e70fa..d719cc5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
@@ -24,7 +24,7 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.Random;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
index 00428d5..2f749d4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
@@ -33,7 +33,7 @@ import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.types.LongValue;
import org.apache.flink.util.Collector;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
index 83ae165..0921e82 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/flink-tests/src/test/java/org/apache/flink/test/util/FailingTestBase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/util/FailingTestBase.java b/flink-tests/src/test/java/org/apache/flink/test/util/FailingTestBase.java
index 09fe150..cc45ec5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/util/FailingTestBase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/util/FailingTestBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.test.util;
-import junit.framework.Assert;
+import org.junit.Assert;
import org.apache.flink.client.minicluster.NepheleMiniCluster;
import org.apache.flink.runtime.client.JobClient;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bef5e548/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 840b1cc..7e0f153 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,7 +77,7 @@ under the License.
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <version>1.1.1</version>
+ <version>1.1.3</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -85,15 +85,7 @@ under the License.
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>1.2.16</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <version>1.3</version>
+ <version>1.2.17</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -101,7 +93,7 @@ under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>17.0</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -109,7 +101,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.7</version>
+ <version>4.11</version>
<type>jar</type>
<scope>test</scope>
</dependency>
@@ -117,7 +109,7 @@ under the License.
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
- <version>1.8.5</version>
+ <version>1.9.5</version>
<type>jar</type>
<scope>test</scope>
</dependency>
@@ -125,7 +117,7 @@ under the License.
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
- <version>1.4.9</version>
+ <version>1.5.5</version>
<type>jar</type>
<scope>test</scope>
</dependency>
@@ -133,7 +125,7 @@ under the License.
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito</artifactId>
- <version>1.4.9</version>
+ <version>1.5.5</version>
<type>jar</type>
<scope>test</scope>
</dependency>
@@ -141,7 +133,7 @@ under the License.
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
- <version>1.1</version>
+ <version>1.3</version>
<type>jar</type>
<scope>test</scope>
</dependency>
@@ -311,6 +303,33 @@ under the License.
<dependencyManagement>
<!-- this section defines the module versions that are used if nothing else is specified. -->
<dependencies>
+ <!-- Make sure we use a consistent jetty version throughout the project -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>8.0.0.M1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>8.0.0.M1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>8.0.0.M1</version>
+ </dependency>
+ <!-- Make sure we use a consistent avro version throughout the project -->
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.7.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-mapred</artifactId>
+ <version>1.7.6</version>
+ </dependency>
<!-- Managed dependency required for HBase in pact-hbase -->
<dependency>
<groupId>org.slf4j</groupId>
[25/39] git commit: [FLINK-701] Several cleanups after SAM
refactoring. - Lambda detection compiles on earlier java versions - Add
lambda detection test. - Fix JavaDocs
Posted by rm...@apache.org.
[FLINK-701] Several cleanups after SAM refactoring.
- Lambda detection compiles on earlier java versions
- Add lambda detection test.
- Fix JavaDocs
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/bc89e911
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/bc89e911
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/bc89e911
Branch: refs/heads/travis_test
Commit: bc89e911b2cd433bb841beab9e8e513ca0c3525d
Parents: 22b24f2
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 31 19:46:14 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 31 20:18:00 2014 +0200
----------------------------------------------------------------------
.../examples/SpargelConnectedComponents.java | 4 +-
.../spargel/java/examples/SpargelPageRank.java | 8 +-
.../SpargelPageRankCountingVertices.java | 11 +-
.../apache/flink/client/web/JobsServlet.java | 1 -
.../org/apache/flink/compiler/PactCompiler.java | 6 +-
.../apache/flink/compiler/CompilerTestBase.java | 6 +-
.../flink/compiler/util/DummyCrossStub.java | 4 -
.../common/functions/AbstractRichFunction.java | 8 +-
.../api/common/functions/CoGroupFunction.java | 38 +++++--
.../api/common/functions/CombineFunction.java | 24 ++++-
.../api/common/functions/CrossFunction.java | 38 +++++--
.../api/common/functions/ExecutionContext.java | 51 ----------
.../api/common/functions/FilterFunction.java | 25 ++++-
.../common/functions/FlatCombineFunction.java | 22 +++-
.../api/common/functions/FlatJoinFunction.java | 45 +++++++-
.../api/common/functions/FlatMapFunction.java | 23 +++--
.../flink/api/common/functions/Function.java | 1 -
.../common/functions/GenericCollectorMap.java | 10 +-
.../common/functions/GroupReduceFunction.java | 34 +++++--
.../api/common/functions/JoinFunction.java | 40 +++++++-
.../flink/api/common/functions/MapFunction.java | 21 +++-
.../api/common/functions/ReduceFunction.java | 26 ++++-
.../api/common/functions/RichFunction.java | 1 -
.../api/common/functions/RuntimeContext.java | 28 -----
.../common/functions/util/FunctionUtils.java | 38 ++++---
.../flink/api/common/operators/Union.java | 1 -
.../operators/base/BulkIterationBase.java | 1 +
.../base/CollectorMapOperatorBase.java | 2 +-
.../api/common/operators/util/OperatorUtil.java | 3 +-
.../common/operators/util/OperatorUtilTest.java | 5 +
.../java/org/apache/flink/api/java/DataSet.java | 49 ++++-----
.../flink/api/java/ExecutionEnvironment.java | 2 +-
.../api/java/functions/FlatMapIterator.java | 2 +-
.../api/java/functions/RichCoGroupFunction.java | 39 +------
.../api/java/functions/RichCrossFunction.java | 35 +------
.../api/java/functions/RichFilterFunction.java | 28 +----
.../java/functions/RichFlatCombineFunction.java | 9 ++
.../java/functions/RichFlatJoinFunction.java | 43 +-------
.../api/java/functions/RichFlatMapFunction.java | 28 +----
.../java/functions/RichGroupReduceFunction.java | 32 +-----
.../api/java/functions/RichJoinFunction.java | 12 ++-
.../api/java/functions/RichMapFunction.java | 29 +-----
.../api/java/functions/RichReduceFunction.java | 35 +------
.../api/java/operators/CoGroupOperator.java | 2 +-
.../flink/api/java/operators/CrossOperator.java | 2 +-
.../api/java/operators/DistinctOperator.java | 28 ++---
.../api/java/operators/GroupReduceOperator.java | 7 +-
.../flink/api/java/operators/Grouping.java | 4 +-
.../flink/api/java/operators/JoinOperator.java | 6 +-
.../api/java/operators/SortedGrouping.java | 4 +-
.../api/java/operators/UnsortedGrouping.java | 2 +-
.../PlanUnwrappingReduceGroupOperator.java | 2 +-
.../java/record/functions/CoGroupFunction.java | 2 +-
.../java/record/functions/CrossFunction.java | 6 +-
.../api/java/record/functions/JoinFunction.java | 15 +--
.../api/java/record/functions/MapFunction.java | 2 +-
.../java/record/functions/ReduceFunction.java | 2 +-
.../api/java/record/operators/MapOperator.java | 1 +
.../DeltaIterationTranslationTest.java | 2 +-
.../translation/DistrinctTranslationTest.java | 57 +++++++++++
.../java/type/extractor/TypeExtractorTest.java | 4 +
.../javaApiOperators/lambdas/CoGroupITCase.java | 1 +
.../javaApiOperators/lambdas/CrossITCase.java | 1 +
.../javaApiOperators/lambdas/FilterITCase.java | 70 ++-----------
.../lambdas/FlatJoinITCase.java | 1 +
.../javaApiOperators/lambdas/FlatMapITCase.java | 1 +
.../lambdas/GroupReduceITCase.java | 1 +
.../javaApiOperators/lambdas/JoinITCase.java | 1 +
.../lambdas/LambdaExtractionTest.java | 82 +++++++++++++++
.../javaApiOperators/lambdas/MapITCase.java | 1 +
.../javaApiOperators/lambdas/ReduceITCase.java | 102 +++++--------------
.../SolutionSetFastUpdateOutputCollector.java | 6 +-
.../io/SolutionSetUpdateOutputCollector.java | 2 +-
.../task/AbstractIterativePactTask.java | 4 +-
.../iterative/task/IterationHeadPactTask.java | 7 +-
.../task/IterationIntermediatePactTask.java | 9 +-
.../task/IterationSynchronizationSinkTask.java | 4 +-
.../iterative/task/IterationTailPactTask.java | 9 +-
.../operators/RuntimeExecutionContext.java | 56 ----------
.../flink/test/operators/CrossITCase.java | 18 +---
80 files changed, 658 insertions(+), 734 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
index a4ba6fa..0f1f26d 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
@@ -18,7 +18,7 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.spargel.java.MessageIterator;
import org.apache.flink.spargel.java.MessagingFunction;
@@ -70,7 +70,7 @@ public class SpargelConnectedComponents {
* A map function that takes a Long value and creates a 2-tuple out of it:
* <pre>(Long value) -> (value, value)</pre>
*/
- public static final class IdAssigner extends RichMapFunction<Long, Tuple2<Long, Long>> {
+ public static final class IdAssigner implements MapFunction<Long, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> map(Long value) {
return new Tuple2<Long, Long>(value, value);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
index 9dfc327..6d3dc95 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
@@ -18,8 +18,8 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.RichFlatMapFunction;
-import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.spargel.java.MessageIterator;
@@ -48,7 +48,7 @@ public class SpargelPageRank {
// enumerate some sample edges and assign an initial uniform probability (rank)
DataSet<Tuple2<Long, Double>> intialRanks = env.generateSequence(1, numVertices)
- .map(new RichMapFunction<Long, Tuple2<Long, Double>>() {
+ .map(new MapFunction<Long, Tuple2<Long, Double>>() {
public Tuple2<Long, Double> map(Long value) {
return new Tuple2<Long, Double>(value, 1.0/numVertices);
}
@@ -56,7 +56,7 @@ public class SpargelPageRank {
// generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, numVertices)
- .flatMap(new RichFlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
int numOutEdges = (int) (Math.random() * (numVertices / 2));
for (int i = 0; i < numOutEdges; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
index 43c0b84..01d2cd7 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
@@ -18,9 +18,10 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.RichMapFunction;
-import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
@@ -53,7 +54,7 @@ public class SpargelPageRankCountingVertices {
// generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, NUM_VERTICES)
- .flatMap(new RichFlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
int numOutEdges = (int) (Math.random() * (NUM_VERTICES / 2));
for (int i = 0; i < numOutEdges; i++) {
@@ -67,12 +68,12 @@ public class SpargelPageRankCountingVertices {
// count the number of vertices
DataSet<Long> count = vertices
- .map(new RichMapFunction<Long, Long>() {
+ .map(new MapFunction<Long, Long>() {
public Long map(Long value) {
return 1L;
}
})
- .reduce(new RichReduceFunction<Long>() {
+ .reduce(new ReduceFunction<Long>() {
public Long reduce(Long value1, Long value2) {
return value1 + value2;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-clients/src/main/java/org/apache/flink/client/web/JobsServlet.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/web/JobsServlet.java b/flink-clients/src/main/java/org/apache/flink/client/web/JobsServlet.java
index c62f0d7..cd53115 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/web/JobsServlet.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/web/JobsServlet.java
@@ -180,7 +180,6 @@ public class JobsServlet extends HttpServlet {
// parse the request
ServletFileUpload uploadHandler = new ServletFileUpload(fileItemFactory);
try {
- @SuppressWarnings("unchecked")
Iterator<FileItem> itr = ((List<FileItem>) uploadHandler.parseRequest(req)).iterator();
// go over the form fields and look for our file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java b/flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java
index b55dea0..e22a365 100644
--- a/flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java
+++ b/flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java
@@ -37,7 +37,6 @@ import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.Union;
import org.apache.flink.api.common.operators.base.BulkIterationBase;
import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.base.CollectorMapOperatorBase;
import org.apache.flink.api.common.operators.base.CrossOperatorBase;
import org.apache.flink.api.common.operators.base.DeltaIterationBase;
import org.apache.flink.api.common.operators.base.FilterOperatorBase;
@@ -648,6 +647,7 @@ public class PactCompiler {
this.forceDOP = forceDOP;
}
+ @SuppressWarnings("deprecation")
@Override
public boolean preVisit(Operator<?> c) {
// check if we have been here before
@@ -671,8 +671,8 @@ public class PactCompiler {
else if (c instanceof MapOperatorBase) {
n = new MapNode((MapOperatorBase<?, ?, ?>) c);
}
- else if (c instanceof CollectorMapOperatorBase) {
- n = new CollectorMapNode((CollectorMapOperatorBase<?, ?, ?>) c);
+ else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
+ n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
}
else if (c instanceof FlatMapOperatorBase) {
n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java b/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
index b2c163b..0115c31 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
@@ -179,7 +179,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
}
@SuppressWarnings("unchecked")
- public <T extends PlanNode> T getNode(String name, Class<? extends RichFunction> stubClass) {
+ public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
List<PlanNode> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
@@ -241,7 +241,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
}
@SuppressWarnings("unchecked")
- public <T extends Operator<?>> T getNode(String name, Class<? extends RichFunction> stubClass) {
+ public <T extends Operator<?>> T getNode(String name, Class<? extends Function> stubClass) {
List<Operator<?>> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
index 736ee14..d6ecc40 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
@@ -16,18 +16,14 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.util;
-import java.io.Serializable;
-
import org.apache.flink.api.java.record.functions.CrossFunction;
import org.apache.flink.types.Record;
public class DummyCrossStub extends CrossFunction {
private static final long serialVersionUID = 1L;
-
@Override
public Record cross(Record first, Record second) throws Exception {
return first;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
index 07b957d..981de14 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
@@ -24,9 +23,10 @@ import java.io.Serializable;
import org.apache.flink.configuration.Configuration;
/**
- * An abstract stub implementation for user-defined functions. It offers default implementations
- * for {@link #open(Configuration)} and {@link #close()}. It also offers access to the
- * {@link RuntimeContext} and {@link IterationRuntimeContext}.
+ * An abstract stub implementation for rich user-defined functions.
+ * Rich functions have additional methods for initialization ({@link #open(Configuration)}) and
+ * teardown ({@link #close()}), as well as access to their runtime execution context via
+ * {@link #getRuntimeContext()}.
*/
public abstract class AbstractRichFunction implements RichFunction, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
index 5c200af..cc8fe78 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
@@ -24,18 +23,41 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
-
+/**
+ * The interface for CoGroup functions. CoGroup functions combine two data sets by first grouping each data set
+ * after a key and then "joining" the groups by calling this function with the two sets for each key.
+ * If a key is present in only one of the two inputs, it may be that one of the groups is empty.
+ * <p>
+ * The basic syntax for using CoGoup on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.coGroup(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyCoGroupFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * <p>
+ * Some keys may only be contained in one of the two original data sets. In that case, the CoGroup function is invoked
+ * with in empty input for the side of the data set that did not contain elements with that specific key.
+ *
+ * @param <V1> The data type of the first input data set.
+ * @param <V2> The data type of the second input data set.
+ * @param <O> The data type of the returned elements.
+ */
public interface CoGroupFunction<V1, V2, O> extends Function, Serializable {
/**
* This method must be implemented to provide a user implementation of a
- * coGroup. It is called for each two key-value pairs that share the same
- * key and come from different inputs.
+ * coGroup. It is called for each pair of element groups where the elements share the
+ * same key.
+ *
+ * @param first The records from the first input.
+ * @param second The records from the second.
+ * @param out A collector to return elements.
*
- * @param first The records from the first input which were paired with the key.
- * @param second The records from the second input which were paired with the key.
- * @param out A collector that collects all output pairs.
+ * @throws Exception The function may throw Exceptions, which will cause the program to cancel,
+ * and may trigger the recovery logic.
*/
void coGroup(Iterator<V1> first, Iterator<V2> second, Collector<O> out) throws Exception;
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
index d72c4c8..aed7fae 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
@@ -16,16 +16,34 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
import java.util.Iterator;
/**
- * Generic interface used for combiners.
+ * Generic interface used for combine functions ("combiners"). Combiners act as auxiliaries to a {@link GroupReduceFunction}
+ * and "pre-reduce" the data. The combine functions typically do not see the entire group of elements, but
+ * only a sub-group.
+ * <p>
+ * Combine functions are frequently helpful in increasing the program efficiency, because they allow the system to
+ * reduce the data volume earlier, before the entire groups have been collected.
+ * <p>
+ * This special variant of the combine function reduces the group of elements into a single element. A variant
+ * that can return multiple values per group is defined in {@link FlatCombineFunction}.
+ *
+ * @param <T> The data type processed by the combine function.
*/
public interface CombineFunction<T> extends Function, Serializable {
- T combine(Iterator<T> records) throws Exception;
+ /**
+ * The combine method, called (potentially multiple timed) with subgroups of elements.
+ *
+ * @param values The elements to be combined.
+ * @return The single resulting value.
+ *
+ * @throws Exception The function may throw Exceptions, which will cause the program to cancel,
+ * and may trigger the recovery logic.
+ */
+ T combine(Iterator<T> values) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
index 0c8bc97..5e2c122 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
@@ -16,27 +16,43 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-
/**
- * @param <IN1> First input type
- * @param <IN2> Second input type
- * @param <OUT> Output type
+ * Interface for Cross functions. Cross functions are applied to the Cartesian produce of their inputs
+ * and call are called for each pair of elements.
+ *
+ * They are optional, a means of convenience the can be used to directly produce manipulate the
+ * pair of elements, instead of processing 2-tuples that contain the pairs.
+ * <p>
+ * The basic syntax for using Cross on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.cross(set2).with(new MyCrossFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
*/
public interface CrossFunction<IN1, IN2, OUT> extends Function, Serializable {
/**
- * User defined function for the cross operator.
+ * Cross UDF method. Called once per pair of elements in the Cartesian product of the inputs.
+ *
+ * @param val1 Element from first input.
+ * @param val2 Element from the second input.
+ * @return The result element.
*
- * @param record1 Record from first input
- * @param record2 Record from the second input
- * @return result of cross UDF.
- * @throws Exception
+ * @throws Exception The function may throw Exceptions, which will cause the program to cancel,
+ * and may trigger the recovery logic.
*/
- OUT cross(IN1 record1, IN2 record2) throws Exception;
+ OUT cross(IN1 val1, IN2 val2) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/ExecutionContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/ExecutionContext.java
deleted file mode 100644
index 9540dc1..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/ExecutionContext.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-
-/**
- * The execution context provides basic information about the parallel runtime context
- * in which a stub instance lives. Such information includes the current number of
- * parallel stub instances, the stub's parallel task index, the pact name, or the iteration context.
- */
-public interface ExecutionContext {
-
- /**
- * Gets the name of the task. This is the name supplied to the contract upon instantiation. If
- * no name was given at contract instantiation time, a default name will be returned.
- *
- * @return The task's name.
- */
- String getTaskName();
-
- /**
- * Gets the number of parallel subtasks in which the stub is executed.
- *
- * @return The number of parallel subtasks in which the stub is executed.
- */
- int getNumberOfSubtasks();
-
- /**
- * Gets the subtask's parallel task number.
- *
- * @return The subtask's parallel task number.
- */
- int getSubtaskIndex();
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
index 2f68477..2380f60 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
@@ -16,18 +16,33 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
+
import java.io.Serializable;
+/**
+ * base interface for Filter functions. A filter function take elements and evaluates a
+ * predicate on them to decide whether to keep the element, or to discard it.
+ * <p>
+ * The basic syntax for using a FilterFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.filter(new MyFilterFunction());
+ * </blockquote></pre>
+ *
+ * @param <T> The type of the filtered elements.
+ */
public interface FilterFunction<T> extends Function, Serializable {
/**
- * User defined function for a filter.
+ * The filter function that evaluates the predicate.
+ *
+ * @param value The value to be filtered.
+ * @return True for values that should be retained, false for values to be filtered out.
*
- * @param value Incoming tuples
- * @return true for tuples that are allowed to pass the filter
- * @throws Exception
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
*/
boolean filter(T value) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
index b2c8f30..edf1614 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
@@ -25,9 +24,28 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
/**
- * Generic interface used for combiners.
+ * Generic interface used for combine functions ("combiners"). Combiners act as auxiliaries to a {@link GroupReduceFunction}
+ * and "pre-reduce" the data. The combine functions typically do not see the entire group of elements, but
+ * only a sub-group.
+ * <p>
+ * Combine functions are frequently helpful in increasing the program efficiency, because they allow the system to
+ * reduce the data volume earlier, before the entire groups have been collected.
+ * <p>
+ * This special variant of the combine function supports to return more than one element per group.
+ * It is frequently less efficient to use than the {@link CombineFunction}.
+ *
+ * @param <T> The data type processed by the combine function.
*/
public interface FlatCombineFunction<T> extends Function, Serializable {
+ /**
+ * The combine method, called (potentially multiple timed) with subgroups of elements.
+ *
+ * @param values The elements to be combined.
+ * @param out The collector to use to return values from the function.
+ *
+ * @throws Exception The function may throw Exceptions, which will cause the program to cancel,
+ * and may trigger the recovery logic.
+ */
void combine(Iterator<T> values, Collector<T> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
index 6a6b971..3ad2c82 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
@@ -16,15 +16,54 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import org.apache.flink.util.Collector;
import java.io.Serializable;
-
+/**
+ * Interface for Join functions. Joins combine two data sets by joining their
+ * elements on specified keys. This function is called with each pair of joining elements.
+ * <p>
+ * This particular variant of the join function supports to return zero, one, or more result values
+ * per pair of joining values.
+ * <p>
+ * By default, the joins follows strictly the semantics of an "inner join" in SQL.
+ * the semantics are those of an "inner join", meaning that elements are filtered out
+ * if their key is not contained in the other data set.
+ * <p>
+ * The basic syntax for using Join on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.join(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyJoinFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * <p>
+ * The Join function is an optional part of a join operation. If no JoinFunction is provided,
+ * the result of the operation is a sequence of 2-tuples, where the elements in the tuple are those that
+ * the JoinFunction would have been invoked with.
+ * <P>
+ * Note: You can use a {@link CoGroupFunction} to perform an outer join.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
public interface FlatJoinFunction<IN1, IN2, OUT> extends Function, Serializable {
- void join (IN1 left, IN2 right, Collector<OUT> out) throws Exception;
+ /**
+ * The join method, called once per joined pair of elements.
+ *
+ * @param first The element from first input.
+ * @param second The element from second input.
+ * @param out The collector used to return zero, one, or more elements.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ void join (IN1 first, IN2 second, Collector<OUT> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
index a8696cf..0c32eae 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
@@ -16,27 +16,36 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import org.apache.flink.util.Collector;
import java.io.Serializable;
-
/**
- *
- * @param <T>
- * @param <O>
+ * interface flatMap functions. FlatMap functions take elements and transform them,
+ * into zero, one, or more elements. Typical applications can be splitting elements, or unnesting lists
+ * and arrays. Operations that produce multiple strictly one result element per input element can also
+ * use the {@link MapFunction}.
+ * <p>
+ * The basic syntax for using a FlatMapFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<Y> result = input.flatMap(new MyFlatMapFunction());
+ * </blockquote></pre>
+ *
+ * @param <T> Type of the input elements.
+ * @param <O> Type of the returned elements.
*/
public interface FlatMapFunction<T, O> extends Function, Serializable {
/**
- * The core method of FlatMappable. Takes an element from the input data set and transforms
+ * The core method of the FlatMapFunction. Takes an element from the input data set and transforms
* it into zero, one, or more elements.
*
* @param value The input value.
- * @param out The collector for for emitting result values.
+ * @param out The collector for returning result values.
*
* @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
* to fail and may trigger recovery.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
index c2a201f..9e9d88d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
@@ -24,5 +24,4 @@ package org.apache.flink.api.common.functions;
* can be called as Java 8 lambdas
*/
public interface Function {
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
index 41cfa1d..edac4a8 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
@@ -16,13 +16,17 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import org.apache.flink.util.Collector;
-
-
+/**
+ * Variant of the flat map that is used for backwards compatibility in the deprecated Record-API-
+ *
+ * @param <T> The input data type.
+ * @param <O> The result data type.
+ */
+@Deprecated
public interface GenericCollectorMap<T, O> extends RichFunction {
void map(T record, Collector<O> out) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
index 984d1fd..5fa959c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
@@ -24,22 +23,35 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
-
/**
- *
- * @param <T> Incoming types
- * @param <O> Outgoing types
+ * The interface for group reduce functions. GroupReduceFunctions process groups of elements.
+ * They may aggregate them to a single value, or produce multiple result values for each group.
+ * The group may be defined by sharing a common grouping key, or the group may simply be
+ * all elements of a data set.
+ * <p>
+ * For a reduce functions that works incrementally by combining always two elements, see
+ * {@link ReduceFunction}.
+ * <p>
+ * The basic syntax for using a grouped GroupReduceFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.groupBy(<key-definition>).reduceGroup(new MyGroupReduceFunction());
+ * </blockquote></pre>
+ *
+ * @param <T> Type of the elements that this function processes.
+ * @param <O> The type of the elements returned by the user-defined function.
*/
public interface GroupReduceFunction<T, O> extends Function, Serializable {
+
/**
+ * The reduce method. The function receives one call per group of elements.
*
- * The central function to be implemented for a reducer. The function receives per call one
- * key and all the values that belong to that key. Each key is guaranteed to be processed by exactly
- * one function call across all involved instances across all computing nodes.
- *
- * @param records All records that belong to the given input key.
+ * @param values All records that belong to the given input key.
* @param out The collector to hand results to.
- * @throws Exception
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
*/
void reduce(Iterator<T> values, Collector<O> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
index 02f526a..9a8943c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
@@ -16,13 +16,49 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-
+/**
+ * Interface for Join functions. Joins combine two data sets by joining their
+ * elements on specified keys. This function is called with each pair of joining elements.
+ * <p>
+ * By default, the joins follows strictly the semantics of an "inner join" in SQL.
+ * the semantics are those of an "inner join", meaning that elements are filtered out
+ * if their key is not contained in the other data set.
+ * <p>
+ * The basic syntax for using Join on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.join(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyJoinFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * <p>
+ * The Join function is an optional part of a join operation. If no JoinFunction is provided,
+ * the result of the operation is a sequence of 2-tuples, where the elements in the tuple are those that
+ * the JoinFunction would have been invoked with.
+ * <P>
+ * Note: You can use a {@link CoGroupFunction} to perform an outer join.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
public interface JoinFunction<IN1, IN2, OUT> extends Function, Serializable {
+ /**
+ * The join method, called once per joined pair of elements.
+ *
+ * @param first The element from first input.
+ * @param second The element from second input.
+ * @return The resulting element.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
OUT join(IN1 first, IN2 second) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
index 4e2520d..dccc980 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
@@ -16,16 +16,31 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
-
import java.io.Serializable;
+/**
+ * Base interface for Map functions. Map functions take elements and transform them,
+ * element wise. A Map function always produces a single result element for each input element.
+ * Typical applications are parsing elements, converting data types, or projecting out fields.
+ * Operations that produce multiple result elements from a single input element can be implemented
+ * using the {@link FlatMapFunction}.
+ * <p>
+ * The basic syntax for using a MapFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<Y> result = input.map(new MyMapFunction());
+ * </blockquote></pre>
+ *
+ * @param <T> Type of the input elements.
+ * @param <O> Type of the returned elements.
+ */
public interface MapFunction<T, O> extends Function, Serializable {
/**
- * The core method of Mappable. Takes an element from the input data set and transforms
+ * The mapping method. Takes an element from the input data set and transforms
* it into exactly one element.
*
* @param value The input value.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
index 04f690a..7a61c97 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
@@ -16,16 +16,36 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
-
import java.io.Serializable;
+/**
+ * Base interface for Reduce functions. Reduce functions combine groups of elements to
+ * a single value, by taking always two elements and combining them into one. Reduce functions
+ * may be used on entire data sets, or on grouped data sets. In the latter case, each group is reduced
+ * individually.
+ * <p>
+ * For a reduce functions that work on an entire group at the same time (such as the
+ * MapReduce/Hadoop-style reduce), see {@link GroupReduceFunction}. In the general case,
+ * ReduceFunctions are considered faster, because they allow the system to use more efficient
+ * execution strategies.
+ * <p>
+ * The basic syntax for using a grouped ReduceFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.groupBy(<key-definition>).reduce(new MyReduceFunction());
+ * </blockquote></pre>
+ * <p>
+ * Like all functions, the ReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <T> Type of the elements that this function processes.
+ */
public interface ReduceFunction<T> extends Function, Serializable {
/**
- * The core method of Reducible, combining two values into one value of the same type.
+ * The core method of ReduceFunction, combining two values into one value of the same type.
* The reduce function is consecutively applied to all values of a group until only a single value remains.
*
* @param value1 The first value to combine.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
index ffc3ac2..8f53252 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
import org.apache.flink.configuration.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index 3cf30ff..e18858b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -105,34 +105,6 @@ public interface RuntimeContext {
* Convenience function to create a counter object for histograms.
*/
Histogram getHistogram(String name);
-
-// /**
-// * I propose to remove this and only keep the other more explicit functions
-// * (to add or get an accumulator object)
-// *
-// * Get an existing or new named accumulator object. Use this function to get
-// * an counter for an custom accumulator type. For the integrated
-// * accumulators you better use convenience functions (e.g. getIntCounter).
-// *
-// * There is no need to register accumulators - they will be created when a
-// * UDF asks the first time for a counter that does not exist yet locally.
-// * This implies that there can be conflicts when a counter is requested with
-// * the same name but with different types, either in the same UDF or in
-// * different. In the last case the conflict occurs during merging.
-// *
-// * @param name
-// * @param accumulatorClass
-// * If the accumulator was not created previously
-// * @return
-// */
-// <V, A> Accumulator<V, A> getAccumulator(String name,
-// Class<? extends Accumulator<V, A>> accumulatorClass);
-//
-// /**
-// * See getAccumulable
-// */
-// <T> SimpleAccumulator<T> getSimpleAccumulator(String name,
-// Class<? extends SimpleAccumulator<T>> accumulatorClass);
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
index bc4ffd0..6455c77 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
@@ -18,18 +18,15 @@
package org.apache.flink.api.common.functions.util;
-
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
-import java.lang.invoke.SerializedLambda;
import java.lang.reflect.Method;
public class FunctionUtils {
-
public static void openFunction (Function function, Configuration parameters) throws Exception{
if (function instanceof RichFunction) {
RichFunction richFunction = (RichFunction) function;
@@ -61,21 +58,30 @@ public class FunctionUtils {
}
}
- public static boolean isSerializedLambdaFunction(Function function) {
- Class<?> clazz = function.getClass();
- try {
- Method replaceMethod = clazz.getDeclaredMethod("writeReplace");
- replaceMethod.setAccessible(true);
- Object serializedForm = replaceMethod.invoke(function);
- if (serializedForm instanceof SerializedLambda) {
- return true;
+ public static boolean isLambdaFunction(Function function) {
+ if (function == null) {
+ throw new IllegalArgumentException();
+ }
+
+ for (Class<?> clazz = function.getClass(); clazz != null; clazz = clazz.getSuperclass()) {
+ try {
+ Method replaceMethod = clazz.getDeclaredMethod("writeReplace");
+ replaceMethod.setAccessible(true);
+ Object serialVersion = replaceMethod.invoke(function);
+
+ if (serialVersion.getClass().getName().equals("java.lang.invoke.SerializedLambda")) {
+ return true;
+ }
}
- else {
- return false;
+ catch (NoSuchMethodException e) {
+ // thrown if the method is not there. fall through the loop
+ }
+ catch (Throwable t) {
+ // this should not happen, we are not executing any method code.
+ throw new RuntimeException("Error while checking whether function is a lambda.", t);
}
}
- catch (Exception e) {
- return false;
- }
+
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
index c416765..6761c17 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.operators;
import org.apache.flink.api.common.functions.AbstractRichFunction;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
index 66bea7f..c91e6ab 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
@@ -48,6 +48,7 @@ import org.apache.flink.util.Visitor;
/**
*
*/
+@SuppressWarnings("deprecation")
public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractRichFunction> implements IterationOperator {
private static String DEFAULT_NAME = "<Unnamed Bulk Iteration>";
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CollectorMapOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CollectorMapOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CollectorMapOperatorBase.java
index ef00a46..6909dd4 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CollectorMapOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CollectorMapOperatorBase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.operators.base;
import org.apache.flink.api.common.functions.GenericCollectorMap;
@@ -33,6 +32,7 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
*
* @see GenericCollectorMap
*/
+@Deprecated
public class CollectorMapOperatorBase<IN, OUT, FT extends GenericCollectorMap<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
public CollectorMapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
index 7d6495b..2683583 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
@@ -48,6 +48,7 @@ import org.apache.flink.api.common.operators.base.JoinOperatorBase;
/**
* Convenience methods when dealing with {@link Operator}s.
*/
+@SuppressWarnings("deprecation")
public class OperatorUtil {
@SuppressWarnings("rawtypes")
@@ -126,7 +127,7 @@ public class OperatorUtil {
* @param inputs
* all input contracts to this contract
*/
- @SuppressWarnings({ "deprecation", "rawtypes", "unchecked" })
+ @SuppressWarnings({ "rawtypes", "unchecked" })
public static void setInputs(final Operator<?> contract, final List<List<Operator>> inputs) {
if (contract instanceof GenericDataSinkBase) {
if (inputs.size() != 1) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
index 30091ab..88809bb 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
@@ -42,6 +42,7 @@ import org.junit.Test;
/**
* Tests {@link OperatorUtil}.
*/
+@SuppressWarnings("deprecation")
public class OperatorUtilTest {
/**
* Test {@link OperatorUtil#getContractClass(Class)}
@@ -115,13 +116,17 @@ public class OperatorUtilTest {
assertEquals(GenericDataSourceBase.class, result);
}
+ @SuppressWarnings("serial")
static abstract class CoGrouper implements CoGroupFunction<IntValue, IntValue, IntValue> {}
+ @SuppressWarnings("serial")
static abstract class Crosser implements CrossFunction<IntValue, IntValue, IntValue> {}
static abstract class Mapper implements GenericCollectorMap<IntValue, IntValue> {}
+ @SuppressWarnings("serial")
static abstract class Matcher implements FlatJoinFunction<IntValue, IntValue, IntValue> {}
+ @SuppressWarnings("serial")
static abstract class Reducer implements GroupReduceFunction<IntValue, IntValue> {}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index e7199f9..3d1238a 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -50,9 +50,9 @@ import org.apache.flink.api.java.operators.MapOperator;
import org.apache.flink.api.java.operators.ProjectOperator.Projection;
import org.apache.flink.api.java.operators.GroupReduceOperator;
import org.apache.flink.api.java.operators.ReduceOperator;
+import org.apache.flink.api.java.operators.SortedGrouping;
import org.apache.flink.api.java.operators.UnionOperator;
import org.apache.flink.api.java.operators.UnsortedGrouping;
-import org.apache.flink.api.java.record.functions.CrossFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.InputTypeConfigurable;
@@ -65,8 +65,8 @@ import org.apache.flink.types.TypeInformation;
* A DataSet represents a collection of elements of the same type.<br/>
* A DataSet can be transformed into another DataSet by applying a transformation as for example
* <ul>
- * <li>{@link DataSet#map(org.apache.flink.api.java.functions.RichMapFunction)},</li>
- * <li>{@link DataSet#reduce(org.apache.flink.api.java.functions.RichReduceFunction)},</li>
+ * <li>{@link DataSet#map(org.apache.flink.api.common.functions.MapFunction)},</li>
+ * <li>{@link DataSet#reduce(org.apache.flink.api.common.functions.ReduceFunction)},</li>
* <li>{@link DataSet#join(DataSet)}, or</li>
* <li>{@link DataSet#coGroup(DataSet)}.</li>
* </ul>
@@ -135,7 +135,7 @@ public abstract class DataSet<T> {
if (mapper == null) {
throw new NullPointerException("Map function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(mapper)) {
+ if (FunctionUtils.isLambdaFunction(mapper)) {
throw new UnsupportedLambdaExpressionException();
}
return new MapOperator<T, R>(this, mapper);
@@ -157,7 +157,7 @@ public abstract class DataSet<T> {
if (flatMapper == null) {
throw new NullPointerException("FlatMap function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(flatMapper)) {
+ if (FunctionUtils.isLambdaFunction(flatMapper)) {
throw new UnsupportedLambdaExpressionException();
}
return new FlatMapOperator<T, R>(this, flatMapper);
@@ -197,13 +197,13 @@ public abstract class DataSet<T> {
*
* @param fieldIndexes The field indexes of the input tuples that are retained.
* The order of fields in the output tuple corresponds to the order of field indexes.
- * @return A Projection that needs to be converted into a {@link ProjectOperator} to complete the
+ * @return A Projection that needs to be converted into a {@link org.apache.flink.api.java.operators.ProjectOperator} to complete the
* Project transformation by calling {@link Projection#types()}.
*
* @see Tuple
* @see DataSet
* @see Projection
- * @see ProjectOperator
+ * @see org.apache.flink.api.java.operators.ProjectOperator
*/
public Projection<T> project(int... fieldIndexes) {
return new Projection<T>(this, fieldIndexes);
@@ -303,7 +303,7 @@ public abstract class DataSet<T> {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ if (FunctionUtils.isLambdaFunction(reducer)) {
throw new UnsupportedLambdaExpressionException();
}
return new GroupReduceOperator<T, R>(this, reducer);
@@ -366,17 +366,15 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param keyExtractor The KeySelector function which extracts the key values from the DataSet on which it is grouped.
* @return An UnsortedGrouping on which a transformation needs to be applied to obtain a transformed DataSet.
*
* @see KeySelector
- * @see Grouping
* @see UnsortedGrouping
- * @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
* @see org.apache.flink.api.java.operators.GroupReduceOperator
@@ -395,17 +393,15 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param fields One or more field positions on which the DataSet will be grouped.
* @return A Grouping on which a transformation needs to be applied to obtain a transformed DataSet.
*
* @see Tuple
- * @see Grouping
* @see UnsortedGrouping
- * @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
* @see org.apache.flink.api.java.operators.GroupReduceOperator
@@ -424,17 +420,15 @@ public abstract class DataSet<T> {
* <ul>
* <li>{@link UnsortedGrouping#sortGroup(int, org.apache.flink.api.common.operators.Order)} to get a {@link SortedGrouping}.
* <li>{@link UnsortedGrouping#aggregate(Aggregations, int)} to apply an Aggregate transformation.
- * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} to apply a Reduce transformation.
- * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)} to apply a GroupReduce transformation.
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)} to apply a Reduce transformation.
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)} to apply a GroupReduce transformation.
* </ul>
*
* @param fields One or more field expressions on which the DataSet will be grouped.
* @return A Grouping on which a transformation needs to be applied to obtain a transformed DataSet.
*
* @see Tuple
- * @see Grouping
* @see UnsortedGrouping
- * @see SortedGrouping
* @see AggregateOperator
* @see ReduceOperator
* @see org.apache.flink.api.java.operators.GroupReduceOperator
@@ -462,7 +456,6 @@ public abstract class DataSet<T> {
* @return A JoinOperatorSets to continue the definition of the Join transformation.
*
* @see JoinOperatorSets
- * @see JoinOperator
* @see DataSet
*/
public <R> JoinOperatorSets<T, R> join(DataSet<R> other) {
@@ -484,7 +477,6 @@ public abstract class DataSet<T> {
* @return A JoinOperatorSets to continue the definition of the Join transformation.
*
* @see JoinOperatorSets
- * @see JoinOperator
* @see DataSet
*/
public <R> JoinOperatorSets<T, R> joinWithTiny(DataSet<R> other) {
@@ -506,7 +498,6 @@ public abstract class DataSet<T> {
* @return A JoinOperatorSet to continue the definition of the Join transformation.
*
* @see JoinOperatorSets
- * @see JoinOperator
* @see DataSet
*/
public <R> JoinOperatorSets<T, R> joinWithHuge(DataSet<R> other) {
@@ -570,7 +561,7 @@ public abstract class DataSet<T> {
* second input being the second field of the tuple.
*
* <p>
- * Call {@link DefaultCross.with(CrossFunction)} to define a {@link CrossFunction} which is called for
+ * Call {@link org.apache.flink.api.java.operators.CrossOperator.DefaultCross#with(CrossFunction)} to define a {@link CrossFunction} which is called for
* each pair of crossed elements. The CrossFunction returns a exactly one element for each pair of input elements.</br>
*
* @param other The other DataSet with which this DataSet is crossed.
@@ -599,14 +590,15 @@ public abstract class DataSet<T> {
* second input being the second field of the tuple.
*
* <p>
- * Call {@link DefaultCross.with(CrossFunction)} to define a {@link CrossFunction} which is called for
+ * Call {@link DefaultCross#with(org.apache.flink.api.common.functions.CrossFunction)} to define a
+ * {@link org.apache.flink.api.common.functions.CrossFunction} which is called for
* each pair of crossed elements. The CrossFunction returns a exactly one element for each pair of input elements.</br>
*
* @param other The other DataSet with which this DataSet is crossed.
* @return A DefaultCross that returns a Tuple2 for each pair of crossed elements.
*
* @see DefaultCross
- * @see CrossFunction
+ * @see org.apache.flink.api.common.functions.CrossFunction
* @see DataSet
* @see Tuple2
*/
@@ -628,14 +620,15 @@ public abstract class DataSet<T> {
* second input being the second field of the tuple.
*
* <p>
- * Call {@link DefaultCross.with(CrossFunction)} to define a {@link CrossFunction} which is called for
+ * Call {@link DefaultCross#with(org.apache.flink.api.common.functions.CrossFunction)} to define a
+ * {@link org.apache.flink.api.common.functions.CrossFunction} which is called for
* each pair of crossed elements. The CrossFunction returns a exactly one element for each pair of input elements.</br>
*
* @param other The other DataSet with which this DataSet is crossed.
* @return A DefaultCross that returns a Tuple2 for each pair of crossed elements.
*
* @see DefaultCross
- * @see CrossFunction
+ * @see org.apache.flink.api.common.functions.CrossFunction
* @see DataSet
* @see Tuple2
*/
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index ebd1422..af0ea52 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -112,7 +112,7 @@ public abstract class ExecutionEnvironment {
* individually override this value to use a specific degree of parallelism via
* {@link Operator#setParallelism(int)}. Other operations may need to run with a different
* degree of parallelism - for example calling
- * {@link DataSet#reduce(org.apache.flink.api.java.functions.RichReduceFunction)} over the entire
+ * {@link DataSet#reduce(org.apache.flink.api.common.functions.ReduceFunction)} over the entire
* set will insert eventually an operation that runs non-parallel (degree of parallelism of one).
*
* @return The degree of parallelism used by operations, unless they override that value. This method
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
index 012ab57..0a83235 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/FlatMapIterator.java
@@ -23,7 +23,7 @@ import java.util.Iterator;
import org.apache.flink.util.Collector;
/**
- * A variant of the {@link RichFlatMapFunction} that returns elements through an iterator, rather then
+ * A convenience variant of the {@link RichFlatMapFunction} that returns elements through an iterator, rather then
* through a collector. In all other respects, it behaves exactly like the FlatMapFunction.
* <p>
* The function needs to be serializable, as defined in {@link java.io.Serializable}.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
index 8aaaf86..e78c31e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
@@ -22,29 +22,14 @@ import java.util.Iterator;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
/**
- * The abstract base class for CoGroup functions. CoGroup functions combine two data sets by first grouping each data set
- * after a key and then "joining" the groups by calling this function with the two sets for each key.
- * If a key is present in only one of the two inputs, it may be that one of the groups is empty.
- * <p>
- * The basic syntax for using CoGoup on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.coGroup(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyCoGroupFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * The keys can be defined through tuple field positions or key extractors.
- * See {@link org.apache.flink.api.java.operators.Keys} for details.
- * <p>
- * Some keys may only be contained in one of the two original data sets. In that case, the CoGroup function is invoked
- * with in empty input for the side of the data set that did not contain elements with that specific key.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link CoGroupFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN1> The type of the elements in the first input.
* @param <IN2> The type of the elements in the second input.
@@ -54,20 +39,6 @@ public abstract class RichCoGroupFunction<IN1, IN2, OUT> extends AbstractRichFun
private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the CoGroupFunction. This method is called for each pair of groups that have the same
- * key. The elements of the groups are returned by the respective iterators.
- *
- * It is possible that one of the two groups is empty, in which case the respective iterator has no elements.
- *
- * @param first The group from the first input.
- * @param second The group from the second input.
- * @param out The collector through which to return the result elements.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract void coGroup(Iterator<IN1> first, Iterator<IN2> second, Collector<OUT> out) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
index a4e1248..58be279 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
@@ -20,26 +20,13 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.CrossFunction;
-
+import org.apache.flink.api.common.functions.RichFunction;
/**
- * The abstract base class for Cross functions. Cross functions build a Cartesian produce of their inputs
- * and call the function or each pair of elements.
- * They are a means of convenience and can be used to directly produce manipulate the
- * pair of elements, instead of having the operator build 2-tuples, and then using a
- * MapFunction over those 2-tuples.
- * <p>
- * The basic syntax for using Cross on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.cross(set2).with(new MyCrossFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link CrossFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN1> The type of the elements in the first input.
* @param <IN2> The type of the elements in the second input.
@@ -48,19 +35,7 @@ import org.apache.flink.api.common.functions.CrossFunction;
public abstract class RichCrossFunction<IN1, IN2, OUT> extends AbstractRichFunction implements CrossFunction<IN1, IN2, OUT> {
private static final long serialVersionUID = 1L;
-
- /**
- * The core method of the cross operation. The method will be invoked for each pair of elements
- * in the Cartesian product.
- *
- * @param first The element from the first input.
- * @param second The element from the second input.
- * @return The result element.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract OUT cross(IN1 first, IN2 second) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
index e3baa74..9057a0f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
@@ -20,19 +20,13 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.RichFunction;
/**
- * The abstract base class for Filter functions. A filter function take elements and evaluates a
- * predicate on them to decide whether to keep the element, or to discard it.
- * <p>
- * The basic syntax for using a FilterFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.filter(new MyFilterFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the FilterFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link FilterFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <T> The type of the filtered elements.
*/
@@ -40,18 +34,6 @@ public abstract class RichFilterFunction<T> extends AbstractRichFunction impleme
private static final long serialVersionUID = 1L;
- /**
- * The core method of the FilterFunction. The method is called for each element in the input,
- * and determines whether the element should be kept or filtered out. If the method returns true,
- * the element passes the filter and is kept, if the method returns false, the element is
- * filtered out.
- *
- * @param value The input value to be filtered.
- * @return Flag to indicate whether to keep the value (true) or to discard it (false).
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract boolean filter(T value) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
index 8c326c6..97c15ff 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
@@ -21,10 +21,19 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
import java.util.Iterator;
+/**
+ * Rich variant of the {@link FlatCombineFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
+ *
+ * @param <T> The data type of the elements to be combined.
+ */
public abstract class RichFlatCombineFunction<T> extends AbstractRichFunction implements FlatCombineFunction<T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
index 15b4539..6918364 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
@@ -20,36 +20,14 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
/**
- * The abstract base class for Join functions. Join functions combine two data sets by joining their
- * elements on specified keys and calling this function with each pair of joining elements.
- * By default, this follows strictly the semantics of an "inner join" in SQL.
- * the semantics are those of an "inner join", meaning that elements are filtered out
- * if their key is not contained in the other data set.
- * <p>
- * Per the semantics of an inner join, the function is
- * <p>
- * The basic syntax for using Join on two data sets is as follows:
- * <pre><blockquote>
- * DataSet<X> set1 = ...;
- * DataSet<Y> set2 = ...;
- *
- * set1.join(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyJoinFunction());
- * </blockquote></pre>
- * <p>
- * {@code set1} is here considered the first input, {@code set2} the second input.
- * The keys can be defined through tuple field positions or key extractors.
- * See {@link org.apache.flink.api.java.operators.Keys} for details.
- * <p>
- * The Join function is actually not a necessary part of a join operation. If no JoinFunction is provided,
- * the result of the operation is a sequence of Tuple2, where the elements in the tuple are those that
- * the JoinFunction would have been invoked with.
- * <P>
- * Note: You can use a {@link RichCoGroupFunction} to perform an outer join.
- * <p>
- * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link FlatJoinFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN1> The type of the elements in the first input.
* @param <IN2> The type of the elements in the second input.
@@ -59,17 +37,6 @@ public abstract class RichFlatJoinFunction<IN1, IN2, OUT> extends AbstractRichFu
private static final long serialVersionUID = 1L;
- /**
- * The user-defined method for performing transformations after a join.
- * The method is called with matching pairs of elements from the inputs.
- *
- * @param first The element from first input.
- * @param second The element from second input.
- * @return The resulting element.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract void join(IN1 first, IN2 second, Collector<OUT> out) throws Exception;
}
[14/39] git commit: [Docs] Adjust Java API documentation to be in
sync with latest changes
Posted by rm...@apache.org.
[Docs] Adjust Java API documentation to be in sync with latest changes
[ci skip]
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/9d1c49fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/9d1c49fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/9d1c49fb
Branch: refs/heads/travis_test
Commit: 9d1c49fbb497cd10258b0135b17fd68ca2d92ce0
Parents: bcf85c2
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 31 17:25:21 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 31 17:31:29 2014 +0200
----------------------------------------------------------------------
docs/_config.yml | 4 +-
docs/java_api_guide.md | 902 +++++++++-------------------------
docs/java_api_transformations.md | 628 +++++++++++++++++++++++
3 files changed, 872 insertions(+), 662 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/9d1c49fb/docs/_config.yml
----------------------------------------------------------------------
diff --git a/docs/_config.yml b/docs/_config.yml
index e418aab..3f2e410 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -5,7 +5,7 @@
# {{ site.CONFIG_KEY }}
#------------------------------------------------------------------------------
-FLINK_VERSION_STABLE: 0.6-SNAPSHOT # this variable can point to a SNAPSHOT version in the git source.
+FLINK_VERSION_STABLE: 0.6-incubating-SNAPSHOT # this variable can point to a SNAPSHOT version in the git source.
FLINK_VERSION_SHORT: 0.6
FLINK_ISSUES_URL: https://issues.apache.org/jira/browse/FLINK
FLINK_GITHUB_URL: https://github.com/apache/incubator-flink
@@ -32,4 +32,4 @@ redcarpet:
# https://help.github.com/articles/github-flavored-markdown
extensions: ["no_intra_emphasis", "fenced_code_blocks", "autolink",
"tables", "with_toc_data", "strikethrough", "superscript",
- "lax_spacing"]
\ No newline at end of file
+ "lax_spacing"]
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/9d1c49fb/docs/java_api_guide.md
----------------------------------------------------------------------
diff --git a/docs/java_api_guide.md b/docs/java_api_guide.md
index 51b5459..f22b2b7 100644
--- a/docs/java_api_guide.md
+++ b/docs/java_api_guide.md
@@ -1,11 +1,8 @@
---
title: "Java API Programming Guide"
---
-<section id="top"></section>
-Java API
-========
-<section id="introduction">
+<section id="top">
Introduction
------------
@@ -48,7 +45,7 @@ public class WordCountExample {
env.execute("Word Count Example");
}
- public static final class LineSplitter extends FlatMapFunction<String, Tuple2<String, Integer>> {
+ public static class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
@Override
public void flatMap(String line, Collector<Tuple2<String, Integer>> out) {
for (String word : line.split(" ")) {
@@ -67,7 +64,7 @@ Linking with Flink
To write programs with Flink, you need to include Flink’s Java API library in your project.
-The simplest way to do this is to use the [quickstart scripts](java_api_quickstart.html). They create a blank project from a template (a Maven Archetype), which sets up everything for you. To manually create the project, you can use the archetype and create a project by calling:
+The simplest way to do this is to use the [quickstart scripts]({{site.baseurl}}/java_api_quickstart.html). They create a blank project from a template (a Maven Archetype), which sets up everything for you. To manually create the project, you can use the archetype and create a project by calling:
```bash
mvn archetype:generate /
@@ -91,10 +88,13 @@ If you want to add Flink to an existing Maven project, add the following entry t
</dependency>
```
-In order to link against the latest SNAPSHOT versions of the code, please follow [this guide]({{site.baseurl}}/downloads.html/#nightly).
+If you are using Flink together with Hadoop, the version of the dependency may vary depending on the version of Hadoop (or more specifically, HDFS) that you want to use Flink with.
+Please refer to the [downloads page]({{site.baseurl}}/downloads.html) for a list of available versions, and instructions on how to link with custom versions of Hadoop.
+
+In order to link against the latest SNAPSHOT versions of the code, please follow [this guide]({{site.baseurl}}/downloads.html#nightly).
The *flink-clients* dependency is only necessary to invoke the Flink program locally (for example to run it standalone for testing and debugging).
-If you intend to only export the program as a JAR file and [run it on a cluster](cluster_execution.html), you can skip that dependency.
+If you intend to only export the program as a JAR file and [run it on a cluster]({{site.baseurl}}/cluster_execution.html), you can skip that dependency.
[Back to top](#top)
@@ -112,7 +112,7 @@ programs with a `main()` method. Each program consists of the same basic parts:
5. Execute your program.
We will now give an overview of each of those steps but please refer
-to the respective sections for more details. Note that all {% gh_link /flink-java/src/main/java/org/apache/flink/api/java "core classes of the Java API" %} are found in the package `org.apache.flinkapi.java`.
+to the respective sections for more details. Note that all {% gh_link /flink-java/src/main/java/org/apache/flink/api/java "core classes of the Java API" %} are found in the package `org.apache.flink.api.java`.
The `ExecutionEnvironment` is the basis for all Flink programs. You can
obtain one using these static methods on class `ExecutionEnvironment`:
@@ -131,8 +131,8 @@ Typically, you only need to use `getExecutionEnvironment()`, since this
will do the right thing depending on the context: if you are executing
your program inside an IDE or as a regular Java program it will create
a local environment that will execute your program on your local machine. If
-you created a JAR file from you program, and invoke it through the [command line](cli.html)
-or the [web interface](web_client.html),
+you created a JAR file from you program, and invoke it through the [command line]({{site.baseurl}}/cli.html)
+or the [web interface]({{site.baseurl}}/web_client.html),
the Flink cluster manager will
execute your main method and `getExecutionEnvironment()` will return
an execution environment for executing your program on a cluster.
@@ -202,6 +202,7 @@ machine or submit your program for execution on a cluster, depending on
how you created the execution environment.
[Back to top](#top)
+
<section id="lazyeval">
Lazy Evaluation
---------------
@@ -209,6 +210,234 @@ Lazy Evaluation
All Flink programs are executed lazily: When the program's main method is executed, the data loading and transformations do not happen directly. Rather, each operation is created and added to the program's plan. The operations are actually executed when one of the `execute()` methods is invoked on the ExecutionEnvironment object. Whether the program is executed locally or on a cluster depends on the environment of the program.
The lazy evaluation lets you construct sophisticated programs that Flink executes as one holistically planned unit.
+[Back to top](#top)
+
+<section id="transformations">
+Transformations
+---------------
+
+Data transformations transform one or more DataSets into a new DataSet. Programs can combine multiple transformations into
+sophisticated assemblies.
+
+This section gives a brief overview of the available transformations. The [transformations documentation]({{site.baseurl}}/java_api_transformations.html)
+has full description of all transformations with examples.
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-center" style="width: 20%">Transformation</th>
+ <th class="text-center">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><strong>Map</strong></td>
+ <td>
+ <p>Takes one element and produces one element.</p>
+{% highlight java %}
+data.map(new MapFunction<String, Integer>() {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>FlatMap</strong></td>
+ <td>
+ <p>Takes one element and produces zero, one, or more elements.</p>
+{% highlight java %}
+data.flatMap(new FlatMapFunction<String, String>() {
+ public void flatMap(String value, Collector<String> out) {
+ for (String s : value.split(" ")) {
+ out.collect(s);
+ }
+ }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>Filter</strong></td>
+ <td>
+ <p>Evaluates a boolean function for each element and retains those for which the function returns *true*.</p>
+{% highlight java %}
+data.filter(new FilterFunction<Integer>() {
+ public boolean filter(Integer value) { return value > 1000; }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>Reduce</strong></td>
+ <td>
+ <p>Combines a group of elements into a single element by repeatedly combining two elements into one.
+ Reduce may be applied on a full data set, or on a grouped data set.</p>
+{% highlight java %}
+data.reduce(new ReduceFunction<Integer> {
+ public Integer reduce(Integer a, Integer b) { return a + b; }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>ReduceGroup</strong></td>
+ <td>
+ <p>Combines a group of elements into one or more elements. ReduceGroup may be applied on a full data set, or on a grouped data set.</p>
+{% highlight java %}
+data.reduceGroup(new GroupReduceFunction<Integer, Integer> {
+ public void reduceGroup(Iterable<Integer> values, Collector<Integer> out) {
+ int prefixSum = 0;
+ for (Integer i : values) {
+ prefixSum += i;
+ out.collect(prefixSum);
+ }
+ }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>Aggregate</strong></td>
+ <td>
+ <p>Aggregates a group of values into a single value. Aggregation functions can be thought of as built-in reduce functions. Aggregate may be applied on a full data set, or on a grouped data set.</p>
+{% highlight java %}
+DataSet<Tuple3<Integer, String, Double>> input = // [...]
+DataSet<Tuple3<Integer, String, Double>> output = input.aggregate(SUM, 0).and(MIN, 2);
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>ReduceGroup</strong></td>
+ <td>
+ <p>Combines a group of elements into one or more elements. ReduceGroup may be applied on a full data set, or on a grouped data set.</p>
+{% highlight java %}
+data.reduceGroup(new GroupReduceFunction<Integer, Integer> {
+ public void reduceGroup(Iterable<Integer> values, Collector<Integer> out) {
+ int prefixSum = 0;
+ for (Integer i : values) {
+ prefixSum += i;
+ out.collect(prefixSum);
+ }
+ }
+});
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>Aggregate</strong></td>
+ <td>
+ <p>Aggregates a group of values into a single value. Aggregation functions can be thought of as built-in reduce functions. Aggregate may be applied on a full data set, or on a grouped data set.</p>
+{% highlight java %}
+DataSet<Tuple3<Integer, String, Double>> input = // [...]
+DataSet<Tuple3<Integer, String, Double>> output = input.aggregate(SUM, 0).and(MIN, 2);
+{% endhighlight %}
+ </td>
+ </tr>
+
+ </tr>
+ <td><strong>Join</strong></td>
+ <td>
+ <p>Joins two data sets by creating all pairs of elements that are equal on their keys. Optionally uses a JoinFunction to turn the pair of elements into a single element. See [keys](#keys) on how to define join keys.</p>
+{% highlight java %}
+result = input1.join(input2)
+ .where(0) // key of the first input (tuple field 0)
+ .equalTo(1); // key of the second input (tuple field 1)
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>CoGroup</strong></td>
+ <td>
+ <p>The two-dimensional variant of the reduce operation. Groups each input on one or more fields and then joins the groups. The transformation function is called per pair of groups. See [keys](#keys) on how to define coGroup keys.</p>
+{% highlight java %}
+data1.coGroup(data2)
+ .where(0)
+ .equalTo(1)
+ .with(new CoGroupFunction<String, String, String>() {
+ public void coGroup(Iterable<String> in1, Iterable<String> in2, Collector<String> out) {
+ out.collect(...);
+ }
+ });
+{% endhighlight %}
+ </td>
+ </tr>
+
+ <tr>
+ <td><strong>Cross</strong></td>
+ <td>
+ <p>Builds the cartesian product (cross product) of two inputs, creating all pairs of elements. Optionally uses a CrossFunction to turn the pair of elements into a single element</p>
+{% highlight java %}
+DataSet<Integer> data1 = // [...]
+DataSet<String> data2 = // [...]
+DataSet<Tuple2<Integer, String>> result = data1.cross(data2);
+{% endhighlight %}
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Union</strong></td>
+ <td>
+ <p>Produces the union of two data sets. This operation happens implicitly if more than one data set is used for a specific function input.</p>
+{% highlight java %}
+DataSet<String> data1 = // [...]
+DataSet<String> data2 = // [...]
+DataSet<String> result = data1.union(data2);
+{% endhighlight %}
+ </td>
+ </tr>
+ </tbody>
+</table>
+
+----------
+
+The following transformations are available on data sets of Tuples:
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-center" stype="width: 20%">Transformation</th>
+ <th class="text-center">Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><strong>Project</strong></td>
+ <td>
+ <p>Selects a subset of fields from the tuples</p>
+{% highlight java %}
+DataSet<Tuple3<Integer, Double, String>> in = // [...]
+DataSet<Tuple2<String, Integer>> out = in.project(2,0).types(String.class, Integer.class);
+{% endhighlight %}
+ </td>
+ </tr>
+ </tbody>
+</table>
+[Back to top](#top)
+
+
+<section id="keys">
+Defining Keys
+-------------
+
+
+[Back to top](#top)
+
+
+<section id="functions">
+Functions
+---------
+
+
+[Back to top](#top)
+
<section id="types">
Data Types
@@ -248,7 +477,6 @@ public class WordWithCount {
You can use all of those types to parameterize `DataSet` and function implementations, e.g. `DataSet<String>` for a `String` data set or `MapFunction<String, Integer>` for a mapper from `String` to `Integer`.
```java
-
// using a basic data type
DataSet<String> numbers = env.fromElements("1", "2");
@@ -341,652 +569,6 @@ The {% gh_link /flink-java/src/main/java/org/apache/flink/api/java/typeutils/Res
[Back to top](#top)
-<section id="transformations">
-Data Transformations
---------------------
-
-A data transformation transforms one or more `DataSet`s into a new `DataSet`. Advanced data analysis programs can be assembled by chaining multiple transformations.
-
-### Map
-
-The Map transformation applies a user-defined `MapFunction` on each element of a DataSet.
-It implements a one-to-one mapping, that is, exactly one element must be returned by
-the function.
-
-The following code transforms a `DataSet` of Integer pairs into a `DataSet` of Integers:
-
-```java
-// MapFunction that adds two integer values
-public class IntAdder extends MapFunction<Tuple2<Integer, Integer>, Integer> {
- @Override
- public Integer map(Tuple2<Integer, Integer> in) {
- return in.f0 + in.f1;
- }
-}
-
-// [...]
-DataSet<Tuple2<Integer, Integer>> intPairs = // [...]
-DataSet<Integer> intSums = intPairs.map(new IntAdder());
-```
-
-### FlatMap
-
-The FlatMap transformation applies a user-defined `FlatMapFunction` on each element of a `DataSet`.
-This variant of a map function can return arbitrary many result elements (including none) for each input element.
-
-The following code transforms a `DataSet` of text lines into a `DataSet` of words:
-
-```java
-// FlatMapFunction that tokenizes a String by whitespace characters and emits all String tokens.
-public class Tokenizer extends FlatMapFunction<String, String> {
- @Override
- public void flatMap(String value, Collector<String> out) {
- for (String token : value.split("\\W")) {
- out.collect(token);
- }
- }
-}
-
-// [...]
-DataSet<String> textLines = // [...]
-DataSet<String> words = textLines.flatMap(new Tokenizer());
-
-```
-
-### Filter
-
-The Filter transformation applies a user-defined `FilterFunction` on each element of a `DataSet` and retains only those elements for which the function returns `true`.
-
-The following code removes all Integers smaller than zero from a `DataSet`:
-
-```java
-// FilterFunction that filters out all Integers smaller than zero.
-public class NaturalNumberFilter extends FilterFunction<Integer> {
- @Override
- public boolean filter(Integer number) {
- return number >= 0;
- }
-}
-
-// [...]
-DataSet<Integer> intNumbers = // [...]
-DataSet<Integer> naturalNumbers = intNumbers.filter(new NaturalNumberFilter());
-```
-
-### Project (Tuple DataSets only)
-
-The Project transformation removes or moves `Tuple` fields of a `Tuple` `DataSet`.
-The `project(int...)` method selects `Tuple` fields that should be retained by their index and defines their order in the output `Tuple`.
-The `types(Class<?> ...)`method must give the types of the output `Tuple` fields.
-
-Projections do not require the definition of a user function.
-
-The following code shows different ways to apply a Project transformation on a `DataSet`:
-
-```java
-DataSet<Tuple3<Integer, Double, String>> in = // [...]
-// converts Tuple3<Integer, Double, String> into Tuple2<String, Integer>
-DataSet<Tuple2<String, Integer>> out = in.project(2,0).types(String.class, Integer.class);
-```
-
-### Transformations on grouped DataSet
-
-The reduce operations can operate on grouped data sets. Specifying the key to
-be used for grouping can be done in two ways:
-
-- a `KeySelector` function or
-- one or more field position keys (`Tuple` `DataSet` only).
-
-Please look at the reduce examples to see how the grouping keys are specified.
-
-### Reduce on grouped DataSet
-
-A Reduce transformation that is applied on a grouped `DataSet` reduces each group to a single element using a user-defined `ReduceFunction`.
-For each group of input elements, a `ReduceFunction` successively combines pairs of elements into one element until only a single element for each group remains.
-
-#### Reduce on DataSet grouped by KeySelector Function
-
-A `KeySelector` function extracts a key value from each element of a `DataSet`. The extracted key value is used to group the `DataSet`.
-The following code shows how to group a POJO `DataSet` using a `KeySelector` function and to reduce it with a `ReduceFunction`.
-
-```java
-// some ordinary POJO
-public class WC {
- public String word;
- public int count;
- // [...]
-}
-
-// ReduceFunction that sums Integer attributes of a POJO
-public class WordCounter extends ReduceFunction<WC> {
- @Override
- public WC reduce(WC in1, WC in2) {
- return new WC(in1.word, in1.count + in2.count);
- }
-}
-
-// [...]
-DataSet<WC> words = // [...]
-DataSet<WC> wordCounts = words
- // DataSet grouping with inline-defined KeySelector function
- .groupBy(
- new KeySelector<WC, String>() {
- public String getKey(WC wc) { return wc.word; }
- })
- // apply ReduceFunction on grouped DataSet
- .reduce(new WordCounter());
-```
-
-#### Reduce on DataSet grouped by Field Position Keys (Tuple DataSets only)
-
-Field position keys specify one or more fields of a `Tuple` `DataSet` that are used as grouping keys.
-The following code shows how to use field position keys and apply a `ReduceFunction`.
-
-```java
-DataSet<Tuple3<String, Integer, Double>> tuples = // [...]
-DataSet<Tuple3<String, Integer, Double>> reducedTuples =
- tuples
- // group DataSet on first and second field of Tuple
- .groupBy(0,1)
- // apply ReduceFunction on grouped DataSet
- .reduce(new MyTupleReducer());
-```
-
-### GroupReduce on grouped DataSet
-
-A GroupReduce transformation that is applied on a grouped `DataSet` calls a user-defined `GroupReduceFunction` for each group. The difference
-between this and `Reduce` is that the user defined function gets the whole group at once.
-The function is invoked with an iterator over all elements of a group and can return an arbitrary number of result elements using the collector.
-
-#### GroupReduce on DataSet grouped by Field Position Keys (Tuple DataSets only)
-
-The following code shows how duplicate strings can be removed from a `DataSet` grouped by Integer.
-
-```java
-public class DistinctReduce
- extends GroupReduceFunction<Tuple2<Integer, String>, Tuple2<Integer, String>> {
- // Set to hold all unique strings of a group
- Set<String> uniqStrings = new HashSet<String>();
-
- @Override
- public void reduce(Iterator<Tuple2<Integer, String>> in, Collector<Tuple2<Integer, String>> out) {
- // clear set
- uniqStrings.clear();
- // there is at least one element in the iterator
- Tuple2<Integer, String> first = in.next();
- Integer key = first.f0;
- uniqStrings.add(first.f1);
- // add all strings of the group to the set
- while(in.hasNext()) {
- uniqStrings.add(in.next().f1);
- }
- // emit all unique strings
- Tuple2<Integer, String> t = new Tuple2<Integer, String>(key, "");
- for(String s : uniqStrings) {
- t.f1 = s;
- out.collect(t);
- }
- }
-}
-
-// [...]
-DataSet<Tuple2<Integer, String>> input = // [...]
-DataSet<Tuple2<Integer, String>> output =
- input
- // group DataSet by the first tuple field
- .groupBy(0)
- // apply GroupReduceFunction on each group and
- // remove elements with duplicate strings.
- .reduceGroup(new DistinctReduce());
-```
-
-**Note:** Flink internally works a lot with mutable objects. Collecting objects like in the above example only works because Strings are immutable in Java!
-
-#### GroupReduce on DataSet grouped by KeySelector Function
-
-Works analogous to `KeySelector` functions in Reduce transformations.
-
-#### GroupReduce on sorted groups (Tuple DataSets only)
-
-A `GroupReduceFunction` accesses the elements of a group using an iterator. Optionally, the iterator can hand out the elements of a group in a specified order. In many cases this can help to reduce the complexity of a user-defined `GroupReduceFunction` and improve its efficiency.
-Right now, this feature is only available for `Tuple` `DataSet`.
-
-The following code shows another example how to remove duplicate Strings in a `DataSet` grouped by an Integer and sorted by String.
-
-```java
-// GroupReduceFunction that removes consecutive identical elements
-public class DistinctReduce
- extends GroupReduceFunction<Tuple2<Integer, String>, Tuple2<Integer, String>> {
- @Override
- public void reduce(Iterator<Tuple2<Integer, String>> in, Collector<Tuple2<Integer, String>> out) {
- // there is at least one element in the iterator
- Tuple2<Integer, String> first = in.next();
- Integer key = first.f0;
- String comp = first.f1;
- // for each element in group
- while(in.hasNext()) {
- String next = in.next().f1;
- // check if strings are different
- if(!next.equals(comp)) {
- // emit a new element
- out.collect(new Tuple2<Integer, String>(key, comp));
- // update compare string
- comp = next;
- }
- }
- // emit last element
- out.collect(new Tuple2<Integer, String>(key, comp));
- }
-}
-
-// [...]
-DataSet<Tuple2<Integer, String>> input = // [...]
-DataSet<Double> output = input
- // group DataSet by the first tuple field
- .groupBy(0)
- // sort groups on second tuple field
- .sortGroup(1, Order.ASCENDING)
- // // apply GroupReduceFunction on DataSet with sorted groups
- .reduceGroup(new DistinctReduce());
-```
-
-**Note:** A GroupSort often comes for free if the grouping is established using a sort-based execution strategy of an operator before the reduce operation.
-
-#### Combinable GroupReduceFunctions
-
-In contrast to a `ReduceFunction`, a `GroupReduceFunction` is not implicitly combinable. In order to make a `GroupReduceFunction` combinable, you need to implement (override) the ```combine()``` method and annotate the `GroupReduceFunction` with the ```@Combinable``` annotation as shown here:
-
-The following code shows how to compute multiple sums using a combinable `GroupReduceFunction`:
-
-```java
-// Combinable GroupReduceFunction that computes two sums.
-@Combinable
-public class MyCombinableGroupReducer
- extends GroupReduceFunction<Tuple3<String, Integer, Double>,
- Tuple3<String, Integer, Double>> {
- @Override
- public void reduce(Iterator<Tuple3<String, Integer, Double>> in,
- Collector<Tuple3<String, Integer, Double>> out) {
- // one element is always present in iterator
- Tuple3<String, Integer, Double> curr = in.next();
- String key = curr.f0;
- int intSum = curr.f1;
- double doubleSum = curr.f2;
- // sum up all ints and doubles
- while(in.hasNext()) {
- curr = in.next();
- intSum += curr.f1;
- doubleSum += curr.f2;
- }
- // emit a tuple with both sums
- out.collect(new Tuple3<String, Integer, Double>(key, intSum, doubleSum));
- }
-
- @Override
- public void combine(Iterator<Tuple3<String, Integer, Double>> in,
- Collector<Tuple3<String, Integer, Double>> out)) {
- // in some cases combine() calls can simply be forwarded to reduce().
- this.reduce(in, out);
- }
-}
-```
-
-### Aggregate on grouped Tuple DataSet
-
-There are some common aggregation operations that are frequently used. The Aggregate transformation provides the following build-in aggregation functions:
-
-- Sum,
-- Min, and
-- Max.
-
-The Aggregate transformation can only be applied on a `Tuple` `DataSet` and supports only field positions keys for grouping.
-
-The following code shows how to apply an Aggregation transformation on a `DataSet` grouped by field position keys:
-
-```java
-DataSet<Tuple3<Integer, String, Double>> input = // [...]
-DataSet<Tuple3<Integer, String, Double>> output = input
- // group DataSet on second field
- .groupBy(1)
- // compute sum of the first field
- .aggregate(SUM, 0)
- // compute minimum of the third field
- .and(MIN, 2);
-```
-
-To apply multiple aggregations on a DataSet it is necessary to use the `.and()` function after the first aggregate, that means `.aggregate(SUM, 0).and(MIN, 2)` produces the sum of field 0 and the minimum of field 2 of the original DataSet.
-In contrast to that `.aggregate(SUM, 0).aggregate(MIN, 2)` will apply an aggregation on an aggregation. In the given example it would produce the minimum of field 2 after calculating the sum of field 0 grouped by field 1.
-
-**Note:** The set of aggregation functions will be extended in the future.
-
-### Reduce on full DataSet
-
-The Reduce transformation applies a user-defined `ReduceFunction` to all elements of a `DataSet`.
-The `ReduceFunction` subsequently combines pairs of elements into one element until only a single element remains.
-
-The following code shows how to sum all elements of an Integer `DataSet`:
-
-```java
-// ReduceFunction that sums Integers
-public class IntSummer extends ReduceFunction<Integer> {
- @Override
- public Integer reduce(Integer num1, Integer num2) {
- return num1 + num2;
- }
-}
-
-// [...]
-DataSet<Integer> intNumbers = // [...]
-DataSet<Integer> sum = intNumbers.reduce(new IntSummer());
-```
-
-Reducing a full `DataSet` using the Reduce transformation implies that the final Reduce operation cannot be done in parallel. However, a `ReduceFunction` is automatically combinable such that a Reduce transformation does not limit scalability for most use cases.
-
-### GroupReduce on full DataSet
-
-The GroupReduce transformation applies a user-defined `GroupReduceFunction` on all elements of a `DataSet`.
-A `GroupReduceFunction` can iterate over all elements of `DataSet` and return an arbitrary number of result elements.
-
-The following example shows how to apply a GroupReduce transformation on a full `DataSet`:
-
-```java
-DataSet<Integer> input = // [...]
-// apply a (preferably combinable) GroupReduceFunction to a DataSet
-DataSet<Double> output = input.reduceGroup(new MyGroupReducer());
-```
-
-**Note:** A GroupReduce transformation on a full `DataSet` cannot be done in parallel if the `GroupReduceFunction` is not combinable. Therefore, this can be a very compute intensive operation. See the paragraph on "Combineable `GroupReduceFunction`s" above to learn how to implement a combinable `GroupReduceFunction`.
-
-### Aggregate on full Tuple DataSet
-
-There are some common aggregation operations that are frequently used. The Aggregate transformation provides the following build-in aggregation functions:
-
-- Sum,
-- Min, and
-- Max.
-
-The Aggregate transformation can only be applied on a `Tuple` `DataSet`.
-
-The following code shows how to apply an Aggregation transformation on a full `DataSet`:
-
-```java
-DataSet<Tuple2<Integer, Double>> input = // [...]
-DataSet<Tuple2<Integer, Double>> output = input
- // compute sum of the first field
- .aggregate(SUM, 0)
- // compute minimum of the second field
- .and(MIN, 1);
-```
-
-**Note:** Extending the set of supported aggregation functions is on our roadmap.
-
-### Join
-
-The Join transformation joins two `DataSet`s into one `DataSet`. The elements of both `DataSet`s are joined on one or more keys which can be specified using
-
-- a `KeySelector` function or
-- one or more field position keys (`Tuple` `DataSet` only).
-
-There are a few different ways to perform a Join transformation which are shown in the following.
-
-#### Default Join (Join into Tuple2)
-
-The default Join transformation produces a new `Tuple``DataSet` with two fields. Each tuple holds a joined element of the first input `DataSet` in the first tuple field and a matching element of the second input `DataSet` in the second field.
-
-The following code shows a default Join transformation using field position keys:
-
-```java
-DataSet<Tuple2<Integer, String>> input1 = // [...]
-DataSet<Tuple2<Double, Integer>> input2 = // [...]
-// result dataset is typed as Tuple2
-DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Double, Integer>>>
- result =
- input1.join(input2)
- // key definition on first DataSet using a field position key
- .where(0)
- // key definition of second DataSet using a field position key
- .equalTo(1);
-```
-
-#### Join with JoinFunction
-
-A Join transformation can also call a user-defined `JoinFunction` to process joining tuples.
-A `JoinFunction` receives one element of the first input `DataSet` and one element of the second input `DataSet` and returns exactly one element.
-
-The following code performs a join of `DataSet` with custom java objects and a `Tuple` `DataSet` using `KeySelector` functions and shows how to call a user-defined `JoinFunction`:
-
-```java
-// some POJO
-public class Rating {
- public String name;
- public String category;
- public int points;
-}
-
-// Join function that joins a custom POJO with a Tuple
-public class PointWeighter
- extends JoinFunction<Rating, Tuple2<String, Double>, Tuple2<String, Double>> {
-
- @Override
- public Tuple2<String, Double> join(Rating rating, Tuple2<String, Double> weight) {
- // multiply the points and rating and construct a new output tuple
- return new Tuple2<String, Double>(rating.name, rating.points * weight.f1);
- }
-}
-
-DataSet<Rating> ratings = // [...]
-DataSet<Tuple2<String, Double>> weights = // [...]
-DataSet<Tuple2<String, Double>>
- weightedRatings =
- ratings.join(weights)
- // key definition of first DataSet using a KeySelector function
- .where(new KeySelection<Rating, String>() {
- public String getKey(Rating r) { return r.category; }
- })
- // key definition of second DataSet using a KeySelector function
- .equalTo(new KeySelection<Tuple2<String, Double>, String>() {
- public String getKey(Tuple2<String, Double> t) { return t.f0; }
- })
- // applying the JoinFunction on joining pairs
- .with(new PointWeighter());
-```
-
-#### Join with Projection
-
-A Join transformation can construct result tuples using a projection as shown here:
-
-```java
-DataSet<Tuple3<Integer, Byte, String>> input1 = // [...]
-DataSet<Tuple2<Integer, Double>> input2 = // [...]
-DataSet<Tuple4<Integer, String, Double, Byte>
- result =
- input1.join(input2)
- // key definition on first DataSet using a field position key
- .where(0)
- // key definition of second DataSet using a field position key
- .equalTo(0)
- // select and reorder fields of matching tuples
- .projectFirst(0,2).projectSecond(1).projectFirst(1)
- .types(Integer.class, String.class, Double.class, Byte.class);
-```
-
-`projectFirst(int...)` and `projectSecond(int...)` select the fields of the first and second joined input that should be assembled into an output `Tuple`. The order of indexes defines the order of fields in the output tuple.
-The join projection works also for non-`Tuple` `DataSet`s. In this case, `projectFirst()` or `projectSecond()` must be called without arguments to add a joined element to the output `Tuple`.
-
-#### Join with DataSet Size Hint
-
-In order to guide the optimizer to pick the right execution strategy, you can hint the size of a `DataSet` to join as shown here:
-
-```java
-DataSet<Tuple2<Integer, String>> input1 = // [...]
-DataSet<Tuple2<Integer, String>> input2 = // [...]
-
-DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Integer, String>>>
- result1 =
- // hint that the second DataSet is very small
- input1.joinWithTiny(input2)
- .where(0)
- .equalTo(0);
-
-DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Integer, String>>>
- result2 =
- // hint that the second DataSet is very large
- input1.joinWithHuge(input2)
- .where(0)
- .equalTo(0);
-```
-
-### Cross
-
-The Cross transformation combines two `DataSet`s into one `DataSet`. It builds all pairwise combinations of the elements of both input `DataSet`s, i.e., it builds a Cartesian product.
-The Cross transformation either calls a user-defined `CrossFunction` on each pair of elements or applies a projection. Both modes are shown in the following.
-
-**Note:** Cross is potentially a *very* compute-intensive operation which can challenge even large compute clusters!
-
-#### Cross with User-Defined Function
-
-A Cross transformation can call a user-defined `CrossFunction`. A `CrossFunction` receives one element of the first input and one element of the second input and returns exactly one result element.
-
-The following code shows how to apply a Cross transformation on two `DataSet`s using a `CrossFunction`:
-
-```java
-public class Coord {
- public int id;
- public int x;
- public int y;
-}
-
-// CrossFunction computes the Euclidean distance between two Coord objects.
-public class EuclideanDistComputer
- extends CrossFunction<Coord, Coord, Tuple3<Integer, Integer, Double>> {
-
- @Override
- public Tuple3<Integer, Integer, Double> cross(Coord c1, Coord c2) {
- // compute Euclidean distance of coordinates
- double dist = Math.sqrt(Math.pow(c1.x - c2.x, 2) + Math.pow(c1.y - c2.y, 2));
- return new Tuple3<Integer, Integer, Double>(c1.id, c2.id, dist);
- }
-}
-
-DataSet<Coord> coords1 = // [...]
-DataSet<Coord> coords2 = // [...]
-DataSet<Tuple3<Integer, Integer, Double>>
- distances =
- coords1.cross(coords2)
- // apply CrossFunction
- .with(new EuclideanDistComputer());
-```
-
-#### Cross with Projection
-
-A Cross transformation can also construct result tuples using a projection as shown here:
-
-```java
-DataSet<Tuple3<Integer, Byte, String>> input1 = // [...]
-DataSet<Tuple2<Integer, Double>> input2 = // [...]
-DataSet<Tuple4<Integer, Byte, Integer, Double>
- result =
- input1.cross(input2)
- // select and reorder fields of matching tuples
- .projectSecond(0).projectFirst(1,0).projectSecond(1)
- .types(Integer.class, Byte.class, Integer.class, Double.class);
-```
-
-The field selection in a Cross projection works the same way as in the projection of Join results.
-
-#### Cross with DataSet Size Hint
-
-In order to guide the optimizer to pick the right execution strategy, you can hint the size of a `DataSet` to cross as shown here:
-
-```java
-DataSet<Tuple2<Integer, String>> input1 = // [...]
-DataSet<Tuple2<Integer, String>> input2 = // [...]
-
-DataSet<Tuple4<Integer, String, Integer, String>>
- udfResult =
- // hint that the second DataSet is very small
- input1.crossWithTiny(input2)
- // apply any Cross function (or projection)
- .with(new MyCrosser());
-
-DataSet<Tuple3<Integer, Integer, String>>
- projectResult =
- // hint that the second DataSet is very large
- input1.crossWithHuge(input2)
- // apply a projection (or any Cross function)
- .projectFirst(0,1).projectSecond(1).types(Integer.class, String.class, String.class)
-```
-
-### CoGroup
-
-The CoGroup transformation jointly processes groups of two `DataSet`s. Both `DataSet`s are grouped on a defined key and groups of both `DataSet`s that share the same key are handed together to a user-defined `CoGroupFunction`. If for a specific key only one `DataSet` has a group, the `CoGroupFunction` is called with this group and an empty group.
-A `CoGroupFunction` can separately iterate over the elements of both groups and return an arbitrary number of result elements.
-
-Similar to Reduce, GroupReduce, and Join, keys can be defined using
-
-- a `KeySelector` function or
-- one or more field position keys (`Tuple` `DataSet` only).
-
-#### CoGroup on DataSets grouped by Field Position Keys (Tuple DataSets only)
-
-```java
-// Some CoGroupFunction definition
-class MyCoGrouper
- extends CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Double>, Double> {
- // set to hold unique Integer values
- Set<Integer> ints = new HashSet<Integer>();
-
- @Override
- public void coGroup(Iterator<Tuple2<String, Integer>> iVals,
- Iterator<Tuple2<String, Double>> dVals,
- Collector<Double> out) {
- // clear Integer set
- ints.clear();
- // add all Integer values in group to set
- while(iVals.hasNext()) {
- ints.add(iVals.next().f1);
- }
- // multiply each Double value with each unique Integer values of group
- while(dVals.hasNext()) {
- for(Integer i : ints) {
- out.collect(dVals.next().f1 * i));
- }
- }
- }
-}
-
-// [...]
-DataSet<Tuple2<String, Integer>> iVals = // [...]
-DataSet<Tuple2<String, Double>> dVals = // [...]
-DataSet<Double> output = iVals.coGroup(dVals)
- // group first DataSet on first tuple field
- .where(0)
- // group second DataSet on first tuple field
- .equalTo(0)
- // apply CoGroup function on each pair of groups
- .with(new MyCoGrouper());
-```
-
-#### CoGroup on DataSets grouped by Key Selector Function
-
-Works analogous to key selector functions in Join transformations.
-
-### Union
-
-Produces the union of two `DataSet`s, which have to be of the same type. A union of more than two `DataSet`s can be implemented with multiple union calls, as shown here:
-
-```java
-DataSet<Tuple2<String, Integer>> vals1 = // [...]
-DataSet<Tuple2<String, Integer>> vals2 = // [...]
-DataSet<Tuple2<String, Integer>> vals3 = // [...]
-DataSet<Tuple2<String, Integer>> unioned = vals1.union(vals2)
- .union(vals3);
-```
-
-
-[Back to top](#top)
-
-
<section id="data_sources">
Data Sources
------------
@@ -1175,7 +757,7 @@ List<Tuple2<String, Integer>> outData = new ArrayList<Tuple2<String, Integer>>()
myResult.output(new LocalCollectionOutputFormat(outData));
```
-**Note:** Collection data sources will only work correctly, if the whole program is executed in the same JVM!
+**Note:** Currently, the collection data sink is restricted to local execution, as a debugging tool.
[Back to top](#top)
@@ -1281,7 +863,7 @@ Semantic annotations can be attached to functions through Java annotations, or p
```java
@ConstantFields("1")
-public class DivideFirstbyTwo extends MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
+public class DivideFirstbyTwo implements MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
@Override
public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) {
value.f0 /= 2;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/9d1c49fb/docs/java_api_transformations.md
----------------------------------------------------------------------
diff --git a/docs/java_api_transformations.md b/docs/java_api_transformations.md
new file mode 100644
index 0000000..bd5c45b
--- /dev/null
+++ b/docs/java_api_transformations.md
@@ -0,0 +1,628 @@
+---
+title: "Java API Transformations"
+---
+
+<section id="top">
+DataSet Transformations
+-----------------------
+
+This document gives a deep-dive into the available transformations on DataSets. For a general introduction to the
+Flink Java API, please refer to the [API guide]({{site.baseurl}}/java_api_guide.html)
+
+
+### Map
+
+The Map transformation applies a user-defined `MapFunction` on each element of a DataSet.
+It implements a one-to-one mapping, that is, exactly one element must be returned by
+the function.
+
+The following code transforms a `DataSet` of Integer pairs into a `DataSet` of Integers:
+
+```java
+// MapFunction that adds two integer values
+public class IntAdder implements MapFunction<Tuple2<Integer, Integer>, Integer> {
+ @Override
+ public Integer map(Tuple2<Integer, Integer> in) {
+ return in.f0 + in.f1;
+ }
+}
+
+// [...]
+DataSet<Tuple2<Integer, Integer>> intPairs = // [...]
+DataSet<Integer> intSums = intPairs.map(new IntAdder());
+```
+
+### FlatMap
+
+The FlatMap transformation applies a user-defined `FlatMapFunction` on each element of a `DataSet`.
+This variant of a map function can return arbitrary many result elements (including none) for each input element.
+
+The following code transforms a `DataSet` of text lines into a `DataSet` of words:
+
+```java
+// FlatMapFunction that tokenizes a String by whitespace characters and emits all String tokens.
+public class Tokenizer implements FlatMapFunction<String, String> {
+ @Override
+ public void flatMap(String value, Collector<String> out) {
+ for (String token : value.split("\\W")) {
+ out.collect(token);
+ }
+ }
+}
+
+// [...]
+DataSet<String> textLines = // [...]
+DataSet<String> words = textLines.flatMap(new Tokenizer());
+
+```
+
+### Filter
+
+The Filter transformation applies a user-defined `FilterFunction` on each element of a `DataSet` and retains only those elements for which the function returns `true`.
+
+The following code removes all Integers smaller than zero from a `DataSet`:
+
+```java
+// FilterFunction that filters out all Integers smaller than zero.
+public class NaturalNumberFilter implements FilterFunction<Integer> {
+ @Override
+ public boolean filter(Integer number) {
+ return number >= 0;
+ }
+}
+
+// [...]
+DataSet<Integer> intNumbers = // [...]
+DataSet<Integer> naturalNumbers = intNumbers.filter(new NaturalNumberFilter());
+```
+
+### Project (Tuple DataSets only)
+
+The Project transformation removes or moves `Tuple` fields of a `Tuple` `DataSet`.
+The `project(int...)` method selects `Tuple` fields that should be retained by their index and defines their order in the output `Tuple`.
+The `types(Class<?> ...)`method must give the types of the output `Tuple` fields.
+
+Projections do not require the definition of a user function.
+
+The following code shows different ways to apply a Project transformation on a `DataSet`:
+
+```java
+DataSet<Tuple3<Integer, Double, String>> in = // [...]
+// converts Tuple3<Integer, Double, String> into Tuple2<String, Integer>
+DataSet<Tuple2<String, Integer>> out = in.project(2,0).types(String.class, Integer.class);
+```
+
+### Transformations on grouped DataSet
+
+The reduce operations can operate on grouped data sets. Specifying the key to
+be used for grouping can be done in two ways:
+
+- a `KeySelector` function or
+- one or more field position keys (`Tuple` `DataSet` only).
+
+Please look at the reduce examples to see how the grouping keys are specified.
+
+### Reduce on grouped DataSet
+
+A Reduce transformation that is applied on a grouped `DataSet` reduces each group to a single element using a user-defined `ReduceFunction`.
+For each group of input elements, a `ReduceFunction` successively combines pairs of elements into one element until only a single element for each group remains.
+
+#### Reduce on DataSet grouped by KeySelector Function
+
+A `KeySelector` function extracts a key value from each element of a `DataSet`. The extracted key value is used to group the `DataSet`.
+The following code shows how to group a POJO `DataSet` using a `KeySelector` function and to reduce it with a `ReduceFunction`.
+
+```java
+// some ordinary POJO
+public class WC {
+ public String word;
+ public int count;
+ // [...]
+}
+
+// ReduceFunction that sums Integer attributes of a POJO
+public class WordCounter implements ReduceFunction<WC> {
+ @Override
+ public WC reduce(WC in1, WC in2) {
+ return new WC(in1.word, in1.count + in2.count);
+ }
+}
+
+// [...]
+DataSet<WC> words = // [...]
+DataSet<WC> wordCounts = words
+ // DataSet grouping with inline-defined KeySelector function
+ .groupBy(
+ new KeySelector<WC, String>() {
+ public String getKey(WC wc) { return wc.word; }
+ })
+ // apply ReduceFunction on grouped DataSet
+ .reduce(new WordCounter());
+```
+
+#### Reduce on DataSet grouped by Field Position Keys (Tuple DataSets only)
+
+Field position keys specify one or more fields of a `Tuple` `DataSet` that are used as grouping keys.
+The following code shows how to use field position keys and apply a `ReduceFunction`.
+
+```java
+DataSet<Tuple3<String, Integer, Double>> tuples = // [...]
+DataSet<Tuple3<String, Integer, Double>> reducedTuples =
+ tuples
+ // group DataSet on first and second field of Tuple
+ .groupBy(0,1)
+ // apply ReduceFunction on grouped DataSet
+ .reduce(new MyTupleReducer());
+```
+
+### GroupReduce on grouped DataSet
+
+A GroupReduce transformation that is applied on a grouped `DataSet` calls a user-defined `GroupReduceFunction` for each group. The difference
+between this and `Reduce` is that the user defined function gets the whole group at once.
+The function is invoked with an Iterable over all elements of a group and can return an arbitrary number of result elements using the collector.
+
+#### GroupReduce on DataSet grouped by Field Position Keys (Tuple DataSets only)
+
+The following code shows how duplicate strings can be removed from a `DataSet` grouped by Integer.
+
+```java
+public class DistinctReduce
+ implements GroupReduceFunction<Tuple2<Integer, String>, Tuple2<Integer, String>> {
+
+ @Override
+ public void reduce(Iterable<Tuple2<Integer, String>> in, Collector<Tuple2<Integer, String>> out) {
+
+ Set<String> uniqStrings = new HashSet<String>();
+ Integer key = null;
+
+ // add all strings of the group to the set
+ for (Tuple2<Integer, String> t : in) {
+ key = t.f0;
+ uniqStrings.add(t.f1);
+ }
+
+ // emit all unique strings.
+ for (String s : uniqStrings) {
+ out.collect(new Tuple2<Integer, String>(key, s));
+ }
+ }
+}
+
+// [...]
+DataSet<Tuple2<Integer, String>> input = // [...]
+DataSet<Tuple2<Integer, String>> output = input
+ .groupBy(0) // group DataSet by the first tuple field
+ .reduceGroup(new DistinctReduce()); // apply GroupReduceFunction
+```
+
+#### GroupReduce on DataSet grouped by KeySelector Function
+
+Works analogous to `KeySelector` functions in Reduce transformations.
+
+#### GroupReduce on sorted groups (Tuple DataSets only)
+
+A `GroupReduceFunction` accesses the elements of a group using an Iterable. Optionally, the Iterable can hand out the elements of a group in a specified order. In many cases this can help to reduce the complexity of a user-defined `GroupReduceFunction` and improve its efficiency.
+Right now, this feature is only available for DataSets of Tuples.
+
+The following code shows another example how to remove duplicate Strings in a `DataSet` grouped by an Integer and sorted by String.
+
+```java
+// GroupReduceFunction that removes consecutive identical elements
+public class DistinctReduce
+ implements GroupReduceFunction<Tuple2<Integer, String>, Tuple2<Integer, String>> {
+
+ @Override
+ public void reduce(Iterable<Tuple2<Integer, String>> in, Collector<Tuple2<Integer, String>> out) {
+ Integer key = null;
+ String comp = null;
+
+ for (Tuple2<Integer, String> t : in) {
+ key = t.f0;
+ String next = t.f1;
+
+ // check if strings are different
+ if (com == null || !next.equals(comp)) {
+ out.collect(new Tuple2<Integer, String>(key, next));
+ comp = next;
+ }
+ }
+ }
+}
+
+// [...]
+DataSet<Tuple2<Integer, String>> input = // [...]
+DataSet<Double> output = input
+ .groupBy(0) // group DataSet by first field
+ .sortGroup(1, Order.ASCENDING) // sort groups on second tuple field
+ .reduceGroup(new DistinctReduce());
+```
+
+**Note:** A GroupSort often comes for free if the grouping is established using a sort-based execution strategy of an operator before the reduce operation.
+
+#### Combinable GroupReduceFunctions
+
+In contrast to a `ReduceFunction`, a `GroupReduceFunction` is not necessarily combinable. In order to make a `GroupReduceFunction` combinable, you need to implement (override) the `combine()` method and annotate the `GroupReduceFunction` with the `@Combinable` annotation as shown here:
+
+```java
+// Combinable GroupReduceFunction that computes two sums.
+// Note that we use the RichGroupReduceFunction because it defines the combine method
+@Combinable
+public class MyCombinableGroupReducer
+ extends RichGroupReduceFunction<Tuple3<String, Integer, Double>,
+ Tuple3<String, Integer, Double>> {
+ @Override
+ public void reduce(Iterable<Tuple3<String, Integer, Double>> in,
+ Collector<Tuple3<String, Integer, Double>> out) {
+
+ String key = null
+ int intSum = 0;
+ double doubleSum = 0.0;
+
+ for (Tuple3<String, Integer, Double> curr : in) {
+ key = curr.f0;
+ intSum += curr.f1;
+ doubleSum += curr.f2;
+ }
+ // emit a tuple with both sums
+ out.collect(new Tuple3<String, Integer, Double>(key, intSum, doubleSum));
+ }
+
+ @Override
+ public void combine(Iterable<Tuple3<String, Integer, Double>> in,
+ Collector<Tuple3<String, Integer, Double>> out)) {
+ // in some cases combine() calls can simply be forwarded to reduce().
+ this.reduce(in, out);
+ }
+}
+```
+
+### Aggregate on grouped Tuple DataSet
+
+There are some common aggregation operations that are frequently used. The Aggregate transformation provides the following build-in aggregation functions:
+
+- Sum,
+- Min, and
+- Max.
+
+The Aggregate transformation can only be applied on a `Tuple` `DataSet` and supports only field positions keys for grouping.
+
+The following code shows how to apply an Aggregation transformation on a `DataSet` grouped by field position keys:
+
+```java
+DataSet<Tuple3<Integer, String, Double>> input = // [...]
+DataSet<Tuple3<Integer, String, Double>> output = input
+ .groupBy(1) // group DataSet on second field
+ .aggregate(SUM, 0) // compute sum of the first field
+ .and(MIN, 2); // compute minimum of the third field
+```
+
+To apply multiple aggregations on a DataSet it is necessary to use the `.and()` function after the first aggregate, that means `.aggregate(SUM, 0).and(MIN, 2)` produces the sum of field 0 and the minimum of field 2 of the original DataSet.
+In contrast to that `.aggregate(SUM, 0).aggregate(MIN, 2)` will apply an aggregation on an aggregation. In the given example it would produce the minimum of field 2 after calculating the sum of field 0 grouped by field 1.
+
+**Note:** The set of aggregation functions will be extended in the future.
+
+### Reduce on full DataSet
+
+The Reduce transformation applies a user-defined `ReduceFunction` to all elements of a `DataSet`.
+The `ReduceFunction` subsequently combines pairs of elements into one element until only a single element remains.
+
+The following code shows how to sum all elements of an Integer `DataSet`:
+
+```java
+// ReduceFunction that sums Integers
+public class IntSummer implements ReduceFunction<Integer> {
+ @Override
+ public Integer reduce(Integer num1, Integer num2) {
+ return num1 + num2;
+ }
+}
+
+// [...]
+DataSet<Integer> intNumbers = // [...]
+DataSet<Integer> sum = intNumbers.reduce(new IntSummer());
+```
+
+Reducing a full `DataSet` using the Reduce transformation implies that the final Reduce operation cannot be done in parallel. However, a `ReduceFunction` is automatically combinable such that a Reduce transformation does not limit scalability for most use cases.
+
+### GroupReduce on full DataSet
+
+The GroupReduce transformation applies a user-defined `GroupReduceFunction` on all elements of a `DataSet`.
+A `GroupReduceFunction` can iterate over all elements of `DataSet` and return an arbitrary number of result elements.
+
+The following example shows how to apply a GroupReduce transformation on a full `DataSet`:
+
+```java
+DataSet<Integer> input = // [...]
+// apply a (preferably combinable) GroupReduceFunction to a DataSet
+DataSet<Double> output = input.reduceGroup(new MyGroupReducer());
+```
+
+**Note:** A GroupReduce transformation on a full `DataSet` cannot be done in parallel if the `GroupReduceFunction` is not combinable. Therefore, this can be a very compute intensive operation. See the paragraph on "Combineable `GroupReduceFunction`s" above to learn how to implement a combinable `GroupReduceFunction`.
+
+### Aggregate on full Tuple DataSet
+
+There are some common aggregation operations that are frequently used. The Aggregate transformation provides the following build-in aggregation functions:
+
+- Sum,
+- Min, and
+- Max.
+
+The Aggregate transformation can only be applied on a `Tuple` `DataSet`.
+
+The following code shows how to apply an Aggregation transformation on a full `DataSet`:
+
+```java
+DataSet<Tuple2<Integer, Double>> input = // [...]
+DataSet<Tuple2<Integer, Double>> output = input
+ .aggregate(SUM, 0) // compute sum of the first field
+ .and(MIN, 1); // compute minimum of the second field
+```
+
+**Note:** Extending the set of supported aggregation functions is on our roadmap.
+
+### Join
+
+The Join transformation joins two `DataSet`s into one `DataSet`. The elements of both `DataSet`s are joined on one or more keys which can be specified using
+
+- a `KeySelector` function or
+- one or more field position keys (`Tuple` `DataSet` only).
+
+There are a few different ways to perform a Join transformation which are shown in the following.
+
+#### Default Join (Join into Tuple2)
+
+The default Join transformation produces a new `Tuple``DataSet` with two fields. Each tuple holds a joined element of the first input `DataSet` in the first tuple field and a matching element of the second input `DataSet` in the second field.
+
+The following code shows a default Join transformation using field position keys:
+
+```java
+DataSet<Tuple2<Integer, String>> input1 = // [...]
+DataSet<Tuple2<Double, Integer>> input2 = // [...]
+// result dataset is typed as Tuple2
+DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Double, Integer>>>
+ result = input1.join(input2)
+ .where(0) // key of the first input
+ .equalTo(1); // key of the second input
+```
+
+#### Join with JoinFunction
+
+A Join transformation can also call a user-defined `JoinFunction` to process joining tuples.
+A `JoinFunction` receives one element of the first input `DataSet` and one element of the second input `DataSet` and returns exactly one element.
+
+The following code performs a join of `DataSet` with custom java objects and a `Tuple` `DataSet` using `KeySelector` functions and shows how to call a user-defined `JoinFunction`:
+
+```java
+// some POJO
+public class Rating {
+ public String name;
+ public String category;
+ public int points;
+}
+
+// Join function that joins a custom POJO with a Tuple
+public class PointWeighter
+ implements JoinFunction<Rating, Tuple2<String, Double>, Tuple2<String, Double>> {
+
+ @Override
+ public Tuple2<String, Double> join(Rating rating, Tuple2<String, Double> weight) {
+ // multiply the points and rating and construct a new output tuple
+ return new Tuple2<String, Double>(rating.name, rating.points * weight.f1);
+ }
+}
+
+DataSet<Rating> ratings = // [...]
+DataSet<Tuple2<String, Double>> weights = // [...]
+DataSet<Tuple2<String, Double>>
+ weightedRatings =
+ ratings.join(weights)
+
+ // key of the first input
+ .where(new KeySelection<Rating, String>() {
+ public String getKey(Rating r) { return r.category; }
+ })
+
+ // key of the second input
+ .equalTo(new KeySelection<Tuple2<String, Double>, String>() {
+ public String getKey(Tuple2<String, Double> t) { return t.f0; }
+ })
+
+ // applying the JoinFunction on joining pairs
+ .with(new PointWeighter());
+```
+
+#### Join with Projection
+
+A Join transformation can construct result tuples using a projection as shown here:
+
+```java
+DataSet<Tuple3<Integer, Byte, String>> input1 = // [...]
+DataSet<Tuple2<Integer, Double>> input2 = // [...]
+DataSet<Tuple4<Integer, String, Double, Byte>
+ result =
+ input1.join(input2)
+ // key definition on first DataSet using a field position key
+ .where(0)
+ // key definition of second DataSet using a field position key
+ .equalTo(0)
+ // select and reorder fields of matching tuples
+ .projectFirst(0,2).projectSecond(1).projectFirst(1)
+ .types(Integer.class, String.class, Double.class, Byte.class);
+```
+
+`projectFirst(int...)` and `projectSecond(int...)` select the fields of the first and second joined input that should be assembled into an output `Tuple`. The order of indexes defines the order of fields in the output tuple.
+The join projection works also for non-`Tuple` `DataSet`s. In this case, `projectFirst()` or `projectSecond()` must be called without arguments to add a joined element to the output `Tuple`.
+
+#### Join with DataSet Size Hint
+
+In order to guide the optimizer to pick the right execution strategy, you can hint the size of a `DataSet` to join as shown here:
+
+```java
+DataSet<Tuple2<Integer, String>> input1 = // [...]
+DataSet<Tuple2<Integer, String>> input2 = // [...]
+
+DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Integer, String>>>
+ result1 =
+ // hint that the second DataSet is very small
+ input1.joinWithTiny(input2)
+ .where(0)
+ .equalTo(0);
+
+DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Integer, String>>>
+ result2 =
+ // hint that the second DataSet is very large
+ input1.joinWithHuge(input2)
+ .where(0)
+ .equalTo(0);
+```
+
+### Cross
+
+The Cross transformation combines two `DataSet`s into one `DataSet`. It builds all pairwise combinations of the elements of both input `DataSet`s, i.e., it builds a Cartesian product.
+The Cross transformation either calls a user-defined `CrossFunction` on each pair of elements or applies a projection. Both modes are shown in the following.
+
+**Note:** Cross is potentially a *very* compute-intensive operation which can challenge even large compute clusters!
+
+#### Cross with User-Defined Function
+
+A Cross transformation can call a user-defined `CrossFunction`. A `CrossFunction` receives one element of the first input and one element of the second input and returns exactly one result element.
+
+The following code shows how to apply a Cross transformation on two `DataSet`s using a `CrossFunction`:
+
+```java
+public class Coord {
+ public int id;
+ public int x;
+ public int y;
+}
+
+// CrossFunction computes the Euclidean distance between two Coord objects.
+public class EuclideanDistComputer
+ implements CrossFunction<Coord, Coord, Tuple3<Integer, Integer, Double>> {
+
+ @Override
+ public Tuple3<Integer, Integer, Double> cross(Coord c1, Coord c2) {
+ // compute Euclidean distance of coordinates
+ double dist = sqrt(pow(c1.x - c2.x, 2) + pow(c1.y - c2.y, 2));
+ return new Tuple3<Integer, Integer, Double>(c1.id, c2.id, dist);
+ }
+}
+
+DataSet<Coord> coords1 = // [...]
+DataSet<Coord> coords2 = // [...]
+DataSet<Tuple3<Integer, Integer, Double>>
+ distances =
+ coords1.cross(coords2)
+ // apply CrossFunction
+ .with(new EuclideanDistComputer());
+```
+
+#### Cross with Projection
+
+A Cross transformation can also construct result tuples using a projection as shown here:
+
+```java
+DataSet<Tuple3<Integer, Byte, String>> input1 = // [...]
+DataSet<Tuple2<Integer, Double>> input2 = // [...]
+DataSet<Tuple4<Integer, Byte, Integer, Double>
+ result =
+ input1.cross(input2)
+ // select and reorder fields of matching tuples
+ .projectSecond(0).projectFirst(1,0).projectSecond(1)
+ .types(Integer.class, Byte.class, Integer.class, Double.class);
+```
+
+The field selection in a Cross projection works the same way as in the projection of Join results.
+
+#### Cross with DataSet Size Hint
+
+In order to guide the optimizer to pick the right execution strategy, you can hint the size of a `DataSet` to cross as shown here:
+
+```java
+DataSet<Tuple2<Integer, String>> input1 = // [...]
+DataSet<Tuple2<Integer, String>> input2 = // [...]
+
+DataSet<Tuple4<Integer, String, Integer, String>>
+ udfResult =
+ // hint that the second DataSet is very small
+ input1.crossWithTiny(input2)
+ // apply any Cross function (or projection)
+ .with(new MyCrosser());
+
+DataSet<Tuple3<Integer, Integer, String>>
+ projectResult =
+ // hint that the second DataSet is very large
+ input1.crossWithHuge(input2)
+ // apply a projection (or any Cross function)
+ .projectFirst(0,1).projectSecond(1).types(Integer.class, String.class, String.class)
+```
+
+### CoGroup
+
+The CoGroup transformation jointly processes groups of two `DataSet`s. Both `DataSet`s are grouped on a defined key and groups of both `DataSet`s that share the same key are handed together to a user-defined `CoGroupFunction`. If for a specific key only one `DataSet` has a group, the `CoGroupFunction` is called with this group and an empty group.
+A `CoGroupFunction` can separately iterate over the elements of both groups and return an arbitrary number of result elements.
+
+Similar to Reduce, GroupReduce, and Join, keys can be defined using
+
+- a `KeySelector` function or
+- one or more field position keys (`Tuple` `DataSet` only).
+
+#### CoGroup on DataSets grouped by Field Position Keys (Tuple DataSets only)
+
+```java
+// Some CoGroupFunction definition
+class MyCoGrouper
+ implements CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Double>, Double> {
+
+ @Override
+ public void coGroup(Iterable<Tuple2<String, Integer>> iVals,
+ Iterable<Tuple2<String, Double>> dVals,
+ Collector<Double> out) {
+
+ Set<Integer> ints = new HashSet<Integer>();
+
+ // add all Integer values in group to set
+ for (Tuple2<String, Integer>> val : iVale) {
+ ints.add(val.f1);
+ }
+
+ // multiply each Double value with each unique Integer values of group
+ for (Tuple2<String, Double> val : dVals) {
+ for (Integer i : ints) {
+ out.collect(val.f1 * i);
+ }
+ }
+ }
+}
+
+// [...]
+DataSet<Tuple2<String, Integer>> iVals = // [...]
+DataSet<Tuple2<String, Double>> dVals = // [...]
+DataSet<Double> output = iVals.coGroup(dVals)
+ // group first DataSet on first tuple field
+ .where(0)
+ // group second DataSet on first tuple field
+ .equalTo(0)
+ // apply CoGroup function on each pair of groups
+ .with(new MyCoGrouper());
+```
+
+#### CoGroup on DataSets grouped by Key Selector Function
+
+Works analogous to key selector functions in Join transformations.
+
+### Union
+
+Produces the union of two `DataSet`s, which have to be of the same type. A union of more than two `DataSet`s can be implemented with multiple union calls, as shown here:
+
+```java
+DataSet<Tuple2<String, Integer>> vals1 = // [...]
+DataSet<Tuple2<String, Integer>> vals2 = // [...]
+DataSet<Tuple2<String, Integer>> vals3 = // [...]
+DataSet<Tuple2<String, Integer>> unioned = vals1.union(vals2)
+ .union(vals3);
+```
+
+
+[Back to top](#top)
+
[03/39] git commit: Add ScalaTest to LICENSE file
Posted by rm...@apache.org.
Add ScalaTest to LICENSE file
[ci skip]
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/911f673f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/911f673f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/911f673f
Branch: refs/heads/travis_test
Commit: 911f673fd1218d09b8ecb02fd75a429ff29bc7c3
Parents: c066190
Author: uce <u....@fu-berlin.de>
Authored: Wed Jul 23 09:09:39 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Wed Jul 23 09:09:39 2014 +0200
----------------------------------------------------------------------
LICENSE | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/911f673f/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index ed91114..572b6db 100644
--- a/LICENSE
+++ b/LICENSE
@@ -224,16 +224,16 @@ under the Apache License (v 2.0):
- Apache Commons FileUpload (http://commons.apache.org/fileupload/)
- Apache Commons IO (http://commons.apache.org/io/)
- Apache Log4J (http://logging.apache.org/log4j/1.2/)
- - Apache Avro (http://avro.apache.org/)
+ - Apache Avro (http://avro.apache.org)
- Apache Hadoop (http://hadoop.apache.org)
- Apache Derby (http://db.apache.org/derby/)
- Google Guava (https://code.google.com/p/guava-libraries/)
- - Netty (http://netty.io/)
+ - Netty (http://netty.io)
- Powermock (http://www.powermock.org)
- Javassist (http://www.javassist.org)
- - Jetty Web Container (http://www.eclipse.org/jetty)
- - Amazon Web Services SDK for Java (http://aws.amazon.com/sdkforjava)
-
+ - Jetty Web Container (http://www.eclipse.org/jetty/)
+ - Amazon Web Services SDK for Java (http://aws.amazon.com/sdkforjava/)
+ - ScalaTest (http://www.scalatest.org)
-----------------------------------------------------------------------
Eclipse Public License - v 1.0
[34/39] git commit: [FLINK-909] Remove additional empty (and non
empty for iterative broadcast variables) superstep. [FLINK-945] Fix early
memory release in iterations
Posted by rm...@apache.org.
[FLINK-909] Remove additional empty (and non empty for iterative broadcast variables) superstep.
[FLINK-945] Fix early memory release in iterations
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/76a48df0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/76a48df0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/76a48df0
Branch: refs/heads/travis_test
Commit: 76a48df00f0fc202bdedefb33d576fd9a18a92c0
Parents: 71cccd6
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Aug 7 22:00:51 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Aug 7 22:26:09 2014 +0200
----------------------------------------------------------------------
.../concurrent/SuperstepKickoffLatch.java | 65 +++++
.../concurrent/SuperstepKickoffLatchBroker.java | 32 +++
.../task/AbstractIterativePactTask.java | 41 +--
.../iterative/task/IterationHeadPactTask.java | 21 +-
.../task/IterationIntermediatePactTask.java | 31 ++-
.../iterative/task/IterationTailPactTask.java | 38 +--
.../concurrent/SuperstepKickoffLatchTest.java | 247 +++++++++++++++++++
7 files changed, 404 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
new file mode 100644
index 0000000..b53928c
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+public class SuperstepKickoffLatch {
+
+ private final Object monitor = new Object();
+
+ private int superstepNumber = 1;
+
+ private boolean terminated;
+
+ public void triggerNextSuperstep() {
+ synchronized (monitor) {
+ if (terminated) {
+ throw new IllegalStateException("Already teriminated.");
+ }
+ superstepNumber++;
+ monitor.notifyAll();
+ }
+ }
+
+ public void signalTermination() {
+ synchronized (monitor) {
+ terminated = true;
+ monitor.notifyAll();
+ }
+ }
+
+ public boolean awaitStartOfSuperstepOrTermination(int superstep) throws InterruptedException {
+ while (true) {
+ synchronized (monitor) {
+ if (terminated) {
+ return true;
+ }
+ else if (superstepNumber == superstep) {
+ // reached the superstep. all good!
+ return false;
+ }
+ else if (superstepNumber == superstep - 1) {
+ monitor.wait(2000);
+ }
+ else {
+ throw new IllegalStateException("Error while waiting for start of next superstep. current= " + superstepNumber + " waitingFor=" + superstep);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
new file mode 100644
index 0000000..41f6985
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+public class SuperstepKickoffLatchBroker extends Broker<SuperstepKickoffLatch> {
+
+ private static final SuperstepKickoffLatchBroker INSTANCE = new SuperstepKickoffLatchBroker();
+
+ private SuperstepKickoffLatchBroker() {}
+
+
+ public static Broker<SuperstepKickoffLatch> instance() {
+ return INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 636c492..4c03278 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -210,28 +210,17 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
return this.iterationAggregators;
}
- protected void checkForTerminationAndResetEndOfSuperstepState() throws IOException {
+ protected void verifyEndOfSuperstepState() throws IOException {
// sanity check that there is at least one iterative input reader
if (this.iterativeInputs.length == 0 && this.iterativeBroadcastInputs.length == 0) {
- throw new IllegalStateException();
+ throw new IllegalStateException("Error: Iterative task without a single iterative input.");
}
- // check whether this step ended due to end-of-superstep, or proper close
- boolean anyClosed = false;
- boolean allClosed = true;
-
for (int inputNum : this.iterativeInputs) {
MutableReader<?> reader = this.inputReaders[inputNum];
- if (reader.isInputClosed()) {
- anyClosed = true;
- }
- else {
- // check if reader has reached the end of superstep, or if the operation skipped out early
+ if (!reader.isInputClosed()) {
if (reader.hasReachedEndOfSuperstep()) {
- allClosed = false;
-
- // also reset the end-of-superstep state
reader.startNextSuperstep();
}
else {
@@ -241,11 +230,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
Object o = this.inputSerializers[inputNum].getSerializer().createInstance();
while ((o = inIter.next(o)) != null);
- if (reader.isInputClosed()) {
- anyClosed = true;
- } else {
- allClosed = false;
-
+ if (!reader.isInputClosed()) {
// also reset the end-of-superstep state
reader.startNextSuperstep();
}
@@ -256,28 +241,16 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
for (int inputNum : this.iterativeBroadcastInputs) {
MutableReader<?> reader = this.broadcastInputReaders[inputNum];
- if (reader.isInputClosed()) {
- anyClosed = true;
- }
- else {
- // sanity check that the BC input is at the end of teh superstep
+ if (!reader.isInputClosed()) {
+
+ // sanity check that the BC input is at the end of the superstep
if (!reader.hasReachedEndOfSuperstep()) {
throw new IllegalStateException("An iterative broadcast input has not been fully consumed.");
}
- allClosed = false;
reader.startNextSuperstep();
}
}
-
- // sanity check whether we saw the same state (end-of-superstep or termination) on all inputs
- if (allClosed != anyClosed) {
- throw new IllegalStateException("Inconsistent state: Iteration termination received on some, but not all inputs.");
- }
-
- if (allClosed) {
- requestTermination();
- }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index 797bbb6..3dbd47c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -41,6 +41,8 @@ import org.apache.flink.runtime.iterative.concurrent.SolutionSetBroker;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
import org.apache.flink.runtime.iterative.concurrent.SuperstepBarrier;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
import org.apache.flink.runtime.iterative.event.AllWorkersDoneEvent;
import org.apache.flink.runtime.iterative.event.TerminationEvent;
import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
@@ -134,8 +136,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
private BlockingBackChannel initBackChannel() throws Exception {
/* get the size of the memory available to the backchannel */
- int backChannelMemoryPages = getMemoryManager().computeNumberOfPages(this.config.getRelativeBackChannelMemory
- ());
+ int backChannelMemoryPages = getMemoryManager().computeNumberOfPages(this.config.getRelativeBackChannelMemory());
/* allocate the memory available to the backchannel */
List<MemorySegment> segments = new ArrayList<MemorySegment>();
@@ -220,6 +221,9 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
try {
/* used for receiving the current iteration result from iteration tail */
+ SuperstepKickoffLatch nextStepKickoff = new SuperstepKickoffLatch();
+ SuperstepKickoffLatchBroker.instance().handIn(brokerKey, nextStepKickoff);
+
BlockingBackChannel backChannel = initBackChannel();
SuperstepBarrier barrier = initSuperstepBarrier();
SolutionSetUpdateBarrier solutionSetUpdateBarrier = null;
@@ -316,12 +320,15 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
+ "]"));
}
requestTermination();
+ nextStepKickoff.signalTermination();
} else {
incrementIterationCounter();
String[] globalAggregateNames = barrier.getAggregatorNames();
Value[] globalAggregates = barrier.getAggregates();
aggregatorRegistry.updateGlobalAggregatesAndReset(globalAggregateNames, globalAggregates);
+
+ nextStepKickoff.triggerNextSuperstep();
}
}
@@ -344,12 +351,10 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
// - solution set index
IterationAggregatorBroker.instance().remove(brokerKey);
BlockingBackChannelBroker.instance().remove(brokerKey);
- if (isWorksetIteration) {
- SolutionSetBroker.instance().remove(brokerKey);
- if (waitForSolutionSetUpdate) {
- SolutionSetUpdateBarrierBroker.instance().remove(brokerKey);
- }
- }
+ SuperstepKickoffLatchBroker.instance().remove(brokerKey);
+ SolutionSetBroker.instance().remove(brokerKey);
+ SolutionSetUpdateBarrierBroker.instance().remove(brokerKey);
+
if (solutionSet != null) {
solutionSet.close();
solutionSet = null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index c23eae1..b12e70b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.functions.Function;
import org.apache.flink.runtime.io.network.api.BufferWriter;
import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
import org.apache.flink.runtime.iterative.event.TerminationEvent;
import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
import org.apache.flink.util.Collector;
@@ -78,6 +80,8 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
@Override
public void run() throws Exception {
+
+ SuperstepKickoffLatch nextSuperstepLatch = SuperstepKickoffLatchBroker.instance().get(brokerKey());
while (this.running && !terminationRequested()) {
@@ -88,26 +92,31 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
super.run();
// check if termination was requested
- checkForTerminationAndResetEndOfSuperstepState();
+ verifyEndOfSuperstepState();
if (isWorksetUpdate && isWorksetIteration) {
long numCollected = worksetUpdateOutputCollector.getElementsCollectedAndReset();
worksetAggregator.aggregate(numCollected);
}
-
+
if (log.isInfoEnabled()) {
log.info(formatLogString("finishing iteration [" + currentIteration() + "]"));
}
+
+ // let the successors know that the end of this superstep data is reached
+ sendEndOfSuperstep();
+
+ if (isWorksetUpdate) {
+ // notify iteration head if responsible for workset update
+ worksetBackChannel.notifyOfEndOfSuperstep();
+ }
+
+ boolean terminated = nextSuperstepLatch.awaitStartOfSuperstepOrTermination(currentIteration() + 1);
- if (!terminationRequested()) {
- if (isWorksetUpdate) {
- // notify iteration head if responsible for workset update
- worksetBackChannel.notifyOfEndOfSuperstep();
- }
-
- // send the end-of-superstep
- sendEndOfSuperstep();
-
+ if (terminated) {
+ requestTermination();
+ }
+ else {
incrementIterationCounter();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index 90d732c..0d9c903 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -23,6 +23,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
+import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
import org.apache.flink.runtime.operators.PactTaskContext;
import org.apache.flink.util.Collector;
@@ -95,23 +97,19 @@ public class IterationTailPactTask<S extends Function, OT> extends AbstractItera
@Override
public void run() throws Exception {
+
+ SuperstepKickoffLatch nextSuperStepLatch = SuperstepKickoffLatchBroker.instance().get(brokerKey());
+
while (this.running && !terminationRequested()) {
if (log.isInfoEnabled()) {
log.info(formatLogString("starting iteration [" + currentIteration() + "]"));
}
- try {
- super.run();
- }
- catch (NullPointerException e) {
- boolean terminationRequested = terminationRequested();
- System.out.println("Nullpoint exception when termination requested was " + terminationRequested);
- e.printStackTrace();
- }
+ super.run();
// check if termination was requested
- checkForTerminationAndResetEndOfSuperstepState();
+ verifyEndOfSuperstepState();
if (isWorksetUpdate && isWorksetIteration) {
// aggregate workset update element count
@@ -123,16 +121,20 @@ public class IterationTailPactTask<S extends Function, OT> extends AbstractItera
if (log.isInfoEnabled()) {
log.info(formatLogString("finishing iteration [" + currentIteration() + "]"));
}
+
+ if (isWorksetUpdate) {
+ // notify iteration head if responsible for workset update
+ worksetBackChannel.notifyOfEndOfSuperstep();
+ } else if (isSolutionSetUpdate) {
+ // notify iteration head if responsible for solution set update
+ solutionSetUpdateBarrier.notifySolutionSetUpdate();
+ }
- if (!terminationRequested()) {
- if (isWorksetUpdate) {
- // notify iteration head if responsible for workset update
- worksetBackChannel.notifyOfEndOfSuperstep();
- } else if (isSolutionSetUpdate) {
- // notify iteration head if responsible for solution set update
- solutionSetUpdateBarrier.notifySolutionSetUpdate();
- }
-
+ boolean terminate = nextSuperStepLatch.awaitStartOfSuperstepOrTermination(currentIteration() + 1);
+ if (terminate) {
+ requestTermination();
+ }
+ else {
incrementIterationCounter();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/76a48df0/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
new file mode 100644
index 0000000..3173570
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchTest.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.iterative.concurrent;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SuperstepKickoffLatchTest {
+
+ @Test
+ public void testWaitFromOne() {
+ try {
+ SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+
+ Waiter w = new Waiter(latch, 2);
+ Thread waiter = new Thread(w);
+ waiter.setDaemon(true);
+ waiter.start();
+
+ WatchDog wd = new WatchDog(waiter, 2000);
+ wd.start();
+
+ Thread.sleep(100);
+
+ latch.triggerNextSuperstep();
+
+ wd.join();
+ if (wd.getError() != null) {
+ throw wd.getError();
+ }
+
+ if (w.getError() != null) {
+ throw w.getError();
+ }
+ }
+ catch (Throwable t) {
+ t.printStackTrace();
+ Assert.fail("Error: " + t.getMessage());
+ }
+ }
+
+ @Test
+ public void testWaitAlreadyFulfilled() {
+ try {
+ SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+ latch.triggerNextSuperstep();
+
+ Waiter w = new Waiter(latch, 2);
+ Thread waiter = new Thread(w);
+ waiter.setDaemon(true);
+ waiter.start();
+
+ WatchDog wd = new WatchDog(waiter, 2000);
+ wd.start();
+
+ Thread.sleep(100);
+
+ wd.join();
+ if (wd.getError() != null) {
+ throw wd.getError();
+ }
+
+ if (w.getError() != null) {
+ throw w.getError();
+ }
+ }
+ catch (Throwable t) {
+ t.printStackTrace();
+ Assert.fail("Error: " + t.getMessage());
+ }
+ }
+
+ @Test
+ public void testWaitIncorrect() {
+ try {
+ SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+ latch.triggerNextSuperstep();
+ latch.triggerNextSuperstep();
+
+ try {
+ latch.awaitStartOfSuperstepOrTermination(2);
+ Assert.fail("should throw exception");
+ }
+ catch (IllegalStateException e) {
+ // good
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail("Error: " + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testWaitIncorrectAsync() {
+ try {
+ SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+ latch.triggerNextSuperstep();
+ latch.triggerNextSuperstep();
+
+ Waiter w = new Waiter(latch, 2);
+ Thread waiter = new Thread(w);
+ waiter.setDaemon(true);
+ waiter.start();
+
+ WatchDog wd = new WatchDog(waiter, 2000);
+ wd.start();
+
+ Thread.sleep(100);
+
+ wd.join();
+ if (wd.getError() != null) {
+ throw wd.getError();
+ }
+
+ if (w.getError() != null) {
+ if (!(w.getError() instanceof IllegalStateException)) {
+ throw new Exception("wrong exception type " + w.getError());
+ }
+ } else {
+ Assert.fail("should cause exception");
+ }
+ }
+ catch (Throwable t) {
+ t.printStackTrace();
+ Assert.fail("Error: " + t.getMessage());
+ }
+ }
+
+ @Test
+ public void testWaitForTermination() {
+ try {
+ SuperstepKickoffLatch latch = new SuperstepKickoffLatch();
+ latch.triggerNextSuperstep();
+ latch.triggerNextSuperstep();
+
+ Waiter w = new Waiter(latch, 4);
+ Thread waiter = new Thread(w);
+ waiter.setDaemon(true);
+ waiter.start();
+
+ WatchDog wd = new WatchDog(waiter, 2000);
+ wd.start();
+
+ latch.signalTermination();
+
+ wd.join();
+ if (wd.getError() != null) {
+ throw wd.getError();
+ }
+
+ if (w.getError() != null) {
+ throw w.getError();
+ }
+ }
+ catch (Throwable t) {
+ t.printStackTrace();
+ Assert.fail("Error: " + t.getMessage());
+ }
+ }
+
+ private static class Waiter implements Runnable {
+
+ private final SuperstepKickoffLatch latch;
+
+ private final int waitFor;
+
+ private volatile Throwable error;
+
+
+ public Waiter(SuperstepKickoffLatch latch, int waitFor) {
+ this.latch = latch;
+ this.waitFor = waitFor;
+ }
+
+ @Override
+ public void run() {
+ try {
+ latch.awaitStartOfSuperstepOrTermination(waitFor);
+ }
+ catch (Throwable t) {
+ this.error = t;
+ }
+ }
+
+ public Throwable getError() {
+ return error;
+ }
+ }
+
+ private static class WatchDog extends Thread {
+
+ private final Thread toWatch;
+
+ private final long timeOut;
+
+ private volatile Throwable failed;
+
+ public WatchDog(Thread toWatch, long timeout) {
+ setDaemon(true);
+ setName("Watchdog");
+ this.toWatch = toWatch;
+ this.timeOut = timeout;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void run() {
+ try {
+ toWatch.join(timeOut);
+
+ if (toWatch.isAlive()) {
+ this.failed = new Exception("timed out");
+ toWatch.interrupt();
+
+ toWatch.join(2000);
+ if (toWatch.isAlive()) {
+ toWatch.stop();
+ }
+ }
+ }
+ catch (Throwable t) {
+ failed = t;
+ }
+ }
+
+ public Throwable getError() {
+ return failed;
+ }
+ }
+}
[17/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllGroupReduceDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllGroupReduceDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllGroupReduceDriver.java
index da0f222..0727d63 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllGroupReduceDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllGroupReduceDriver.java
@@ -21,8 +21,8 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.runtime.util.MutableToRegularIteratorWrapper;
@@ -37,13 +37,13 @@ import org.apache.flink.util.MutableObjectIterator;
* The GroupReduceTask creates a iterator over all records from its input. The iterator returns all records grouped by their
* key. The iterator is handed to the <code>reduce()</code> method of the GroupReduceFunction.
*
- * @see GenericGroupReduce
+ * @see org.apache.flink.api.common.functions.GroupReduceFunction
*/
-public class AllGroupReduceDriver<IT, OT> implements PactDriver<GenericGroupReduce<IT, OT>, OT> {
+public class AllGroupReduceDriver<IT, OT> implements PactDriver<GroupReduceFunction<IT, OT>, OT> {
private static final Log LOG = LogFactory.getLog(AllGroupReduceDriver.class);
- private PactTaskContext<GenericGroupReduce<IT, OT>, OT> taskContext;
+ private PactTaskContext<GroupReduceFunction<IT, OT>, OT> taskContext;
private MutableObjectIterator<IT> input;
@@ -54,7 +54,7 @@ public class AllGroupReduceDriver<IT, OT> implements PactDriver<GenericGroupRedu
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericGroupReduce<IT, OT>, OT> context) {
+ public void setup(PactTaskContext<GroupReduceFunction<IT, OT>, OT> context) {
this.taskContext = context;
}
@@ -64,9 +64,9 @@ public class AllGroupReduceDriver<IT, OT> implements PactDriver<GenericGroupRedu
}
@Override
- public Class<GenericGroupReduce<IT, OT>> getStubType() {
+ public Class<GroupReduceFunction<IT, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericGroupReduce<IT, OT>> clazz = (Class<GenericGroupReduce<IT, OT>>) (Class<?>) GenericGroupReduce.class;
+ final Class<GroupReduceFunction<IT, OT>> clazz = (Class<GroupReduceFunction<IT, OT>>) (Class<?>) GroupReduceFunction.class;
return clazz;
}
@@ -83,8 +83,8 @@ public class AllGroupReduceDriver<IT, OT> implements PactDriver<GenericGroupRedu
this.strategy = config.getDriverStrategy();
if (strategy == DriverStrategy.ALL_GROUP_COMBINE) {
- if (!(this.taskContext.getStub() instanceof GenericCombine)) {
- throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + GenericCombine.class.getName());
+ if (!(this.taskContext.getStub() instanceof FlatCombineFunction)) {
+ throw new Exception("Using combiner on a UDF that does not implement the combiner interface " + FlatCombineFunction.class.getName());
}
}
else if (strategy != DriverStrategy.ALL_GROUP_REDUCE) {
@@ -105,13 +105,13 @@ public class AllGroupReduceDriver<IT, OT> implements PactDriver<GenericGroupRedu
// single UDF call with the single group
if (inIter.hasNext()) {
if (strategy == DriverStrategy.ALL_GROUP_REDUCE) {
- final GenericGroupReduce<IT, OT> reducer = this.taskContext.getStub();
+ final GroupReduceFunction<IT, OT> reducer = this.taskContext.getStub();
final Collector<OT> output = this.taskContext.getOutputCollector();
reducer.reduce(inIter, output);
}
else {
@SuppressWarnings("unchecked")
- final GenericCombine<IT> combiner = (GenericCombine<IT>) this.taskContext.getStub();
+ final FlatCombineFunction<IT> combiner = (FlatCombineFunction<IT>) this.taskContext.getStub();
@SuppressWarnings("unchecked")
final Collector<IT> output = (Collector<IT>) this.taskContext.getOutputCollector();
combiner.combine(inIter, output);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllReduceDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllReduceDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllReduceDriver.java
index 30bfae3..721f4f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllReduceDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AllReduceDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.runtime.operators.util.TaskConfig;
@@ -35,13 +35,13 @@ import org.apache.flink.util.MutableObjectIterator;
* The ReduceTask creates a iterator over all records from its input. The iterator returns all records grouped by their
* key. The iterator is handed to the <code>reduce()</code> method of the ReduceFunction.
*
- * @see GenericReduce
+ * @see org.apache.flink.api.common.functions.ReduceFunction
*/
-public class AllReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
+public class AllReduceDriver<T> implements PactDriver<ReduceFunction<T>, T> {
private static final Log LOG = LogFactory.getLog(AllReduceDriver.class);
- private PactTaskContext<GenericReduce<T>, T> taskContext;
+ private PactTaskContext<ReduceFunction<T>, T> taskContext;
private MutableObjectIterator<T> input;
@@ -52,7 +52,7 @@ public class AllReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericReduce<T>, T> context) {
+ public void setup(PactTaskContext<ReduceFunction<T>, T> context) {
this.taskContext = context;
this.running = true;
}
@@ -63,9 +63,9 @@ public class AllReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
}
@Override
- public Class<GenericReduce<T>> getStubType() {
+ public Class<ReduceFunction<T>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericReduce<T>> clazz = (Class<GenericReduce<T>>) (Class<?>) GenericReduce.class;
+ final Class<ReduceFunction<T>> clazz = (Class<ReduceFunction<T>>) (Class<?>) ReduceFunction.class;
return clazz;
}
@@ -94,7 +94,7 @@ public class AllReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
LOG.debug(this.taskContext.formatLogString("AllReduce preprocessing done. Running Reducer code."));
}
- final GenericReduce<T> stub = this.taskContext.getStub();
+ final ReduceFunction<T> stub = this.taskContext.getStub();
final MutableObjectIterator<T> input = this.input;
final TypeSerializer<T> serializer = this.serializer;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupDriver.java
index 3da451a..8ff0262 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -41,12 +41,12 @@ import org.apache.flink.util.MutableObjectIterator;
*
* @see org.apache.flink.api.java.record.functions.CoGroupFunction
*/
-public class CoGroupDriver<IT1, IT2, OT> implements PactDriver<GenericCoGrouper<IT1, IT2, OT>, OT> {
+public class CoGroupDriver<IT1, IT2, OT> implements PactDriver<CoGroupFunction<IT1, IT2, OT>, OT> {
private static final Log LOG = LogFactory.getLog(CoGroupDriver.class);
- private PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> taskContext;
+ private PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> taskContext;
private CoGroupTaskIterator<IT1, IT2> coGroupIterator; // the iterator that does the actual cogroup
@@ -56,7 +56,7 @@ public class CoGroupDriver<IT1, IT2, OT> implements PactDriver<GenericCoGrouper<
@Override
- public void setup(PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -69,9 +69,9 @@ public class CoGroupDriver<IT1, IT2, OT> implements PactDriver<GenericCoGrouper<
@Override
- public Class<GenericCoGrouper<IT1, IT2, OT>> getStubType() {
+ public Class<CoGroupFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericCoGrouper<IT1, IT2, OT>> clazz = (Class<GenericCoGrouper<IT1, IT2, OT>>) (Class<?>) GenericCoGrouper.class;
+ final Class<CoGroupFunction<IT1, IT2, OT>> clazz = (Class<CoGroupFunction<IT1, IT2, OT>>) (Class<?>) CoGroupFunction.class;
return clazz;
}
@@ -122,7 +122,7 @@ public class CoGroupDriver<IT1, IT2, OT> implements PactDriver<GenericCoGrouper<
@Override
public void run() throws Exception
{
- final GenericCoGrouper<IT1, IT2, OT> coGroupStub = this.taskContext.getStub();
+ final CoGroupFunction<IT1, IT2, OT> coGroupStub = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
final CoGroupTaskIterator<IT1, IT2> coGroupIterator = this.coGroupIterator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
index 9d06618..8761a2e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
@@ -34,9 +34,9 @@ import org.apache.flink.runtime.util.KeyGroupedIterator;
import org.apache.flink.runtime.util.SingleElementIterator;
import org.apache.flink.util.Collector;
-public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettablePactDriver<GenericCoGrouper<IT1, IT2, OT>, OT> {
+public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettablePactDriver<CoGroupFunction<IT1, IT2, OT>, OT> {
- private PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> taskContext;
+ private PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> taskContext;
private CompactingHashTable<IT1> hashTable;
@@ -53,7 +53,7 @@ public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements Resettab
// --------------------------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -64,9 +64,9 @@ public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements Resettab
}
@Override
- public Class<GenericCoGrouper<IT1, IT2, OT>> getStubType() {
+ public Class<CoGroupFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericCoGrouper<IT1, IT2, OT>> clazz = (Class<GenericCoGrouper<IT1, IT2, OT>>) (Class<?>) GenericCoGrouper.class;
+ final Class<CoGroupFunction<IT1, IT2, OT>> clazz = (Class<CoGroupFunction<IT1, IT2, OT>>) (Class<?>) CoGroupFunction.class;
return clazz;
}
@@ -123,7 +123,7 @@ public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements Resettab
@Override
public void run() throws Exception {
- final GenericCoGrouper<IT1, IT2, OT> coGroupStub = taskContext.getStub();
+ final CoGroupFunction<IT1, IT2, OT> coGroupStub = taskContext.getStub();
final Collector<OT> collector = taskContext.getOutputCollector();
IT1 buildSideRecord = solutionSideRecord;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
index 80fa855..f2020c7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
@@ -34,9 +34,9 @@ import org.apache.flink.runtime.util.KeyGroupedIterator;
import org.apache.flink.runtime.util.SingleElementIterator;
import org.apache.flink.util.Collector;
-public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements ResettablePactDriver<GenericCoGrouper<IT1, IT2, OT>, OT> {
+public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements ResettablePactDriver<CoGroupFunction<IT1, IT2, OT>, OT> {
- private PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> taskContext;
+ private PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> taskContext;
private CompactingHashTable<IT2> hashTable;
@@ -53,7 +53,7 @@ public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resetta
// --------------------------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericCoGrouper<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<CoGroupFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -64,9 +64,9 @@ public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resetta
}
@Override
- public Class<GenericCoGrouper<IT1, IT2, OT>> getStubType() {
+ public Class<CoGroupFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericCoGrouper<IT1, IT2, OT>> clazz = (Class<GenericCoGrouper<IT1, IT2, OT>>) (Class<?>) GenericCoGrouper.class;
+ final Class<CoGroupFunction<IT1, IT2, OT>> clazz = (Class<CoGroupFunction<IT1, IT2, OT>>) (Class<?>) CoGroupFunction.class;
return clazz;
}
@@ -123,7 +123,7 @@ public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resetta
@Override
public void run() throws Exception {
- final GenericCoGrouper<IT1, IT2, OT> coGroupStub = taskContext.getStub();
+ final CoGroupFunction<IT1, IT2, OT> coGroupStub = taskContext.getStub();
final Collector<OT> collector = taskContext.getOutputCollector();
IT2 buildSideRecord = solutionSideRecord;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CrossDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CrossDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CrossDriver.java
index 7c311ed..b68f24d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CrossDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CrossDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericCrosser;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.memorymanager.MemoryManager;
import org.apache.flink.runtime.operators.resettable.BlockResettableMutableObjectIterator;
@@ -40,12 +40,12 @@ import org.apache.flink.util.MutableObjectIterator;
*
* @see org.apache.flink.api.java.functions.CrossFunction
*/
-public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2, OT>, OT> {
+public class CrossDriver<T1, T2, OT> implements PactDriver<CrossFunction<T1, T2, OT>, OT> {
private static final Log LOG = LogFactory.getLog(CrossDriver.class);
- private PactTaskContext<GenericCrosser<T1, T2, OT>, OT> taskContext;
+ private PactTaskContext<CrossFunction<T1, T2, OT>, OT> taskContext;
private MemoryManager memManager;
@@ -67,7 +67,7 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
@Override
- public void setup(PactTaskContext<GenericCrosser<T1, T2, OT>, OT> context) {
+ public void setup(PactTaskContext<CrossFunction<T1, T2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -80,9 +80,9 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
@Override
- public Class<GenericCrosser<T1, T2, OT>> getStubType() {
+ public Class<CrossFunction<T1, T2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericCrosser<T1, T2, OT>> clazz = (Class<GenericCrosser<T1, T2, OT>>) (Class<?>) GenericCrosser.class;
+ final Class<CrossFunction<T1, T2, OT>> clazz = (Class<CrossFunction<T1, T2, OT>>) (Class<?>) CrossFunction.class;
return clazz;
}
@@ -207,7 +207,7 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
final T2 val2Reuse = serializer2.createInstance();
T2 val2Copy = serializer2.createInstance();
- final GenericCrosser<T1, T2, OT> crosser = this.taskContext.getStub();
+ final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
// for all blocks
@@ -217,7 +217,8 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
// for all values in the block
while ((val1 = blockVals.next(val1Reuse)) != null) {
val2Copy = serializer2.copy(val2, val2Copy);
- crosser.cross(val1, val2Copy, collector);
+ collector.collect(crosser.cross(val1,val2Copy));
+ //crosser.cross(val1, val2Copy, collector);
}
blockVals.reset();
}
@@ -254,7 +255,7 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
T2 val2;
final T2 val2Reuse = serializer2.createInstance();
- final GenericCrosser<T1, T2, OT> crosser = this.taskContext.getStub();
+ final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
// for all blocks
@@ -264,7 +265,8 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
// for all values in the block
while (this.running && ((val2 = blockVals.next(val2Reuse)) != null)) {
val1Copy = serializer1.copy(val1, val1Copy);
- crosser.cross(val1Copy, val2, collector);
+ collector.collect(crosser.cross(val1Copy, val2));
+ //crosser.cross(val1Copy, val2, collector);
}
blockVals.reset();
}
@@ -296,7 +298,7 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
T2 val2;
final T2 val2Reuse = serializer2.createInstance();
- final GenericCrosser<T1, T2, OT> crosser = this.taskContext.getStub();
+ final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
// for all blocks
@@ -304,7 +306,8 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
// for all values from the spilling side
while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
val1Copy = serializer1.copy(val1, val1Copy);
- crosser.cross(val1Copy, val2, collector);
+ collector.collect(crosser.cross(val1Copy, val2));
+ //crosser.cross(val1Copy, val2, collector);
}
spillVals.reset();
}
@@ -332,7 +335,7 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
final T2 val2Reuse = serializer2.createInstance();
T2 val2Copy = serializer2.createInstance();
- final GenericCrosser<T1, T2, OT> crosser = this.taskContext.getStub();
+ final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
// for all blocks
@@ -340,7 +343,8 @@ public class CrossDriver<T1, T2, OT> implements PactDriver<GenericCrosser<T1, T2
// for all values from the spilling side
while (this.running && (val1 = spillVals.next(val1Reuse)) != null) {
val2Copy = serializer2.copy(val2, val2Copy);
- crosser.cross(val1, val2Copy, collector);
+ collector.collect(crosser.cross(val1, val2Copy));
+ //crosser.cross(val1, val2Copy, collector);
}
spillVals.reset();
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/FlatMapDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/FlatMapDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/FlatMapDriver.java
index 44f22a0..130601b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/FlatMapDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/FlatMapDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -31,20 +31,20 @@ import org.apache.flink.util.MutableObjectIterator;
* The MapTask creates an iterator over all key-value pairs of its input and hands that to the <code>map()</code> method
* of the MapFunction.
*
- * @see GenericFlatMap
+ * @see org.apache.flink.api.common.functions.FlatMapFunction
*
* @param <IT> The mapper's input data type.
* @param <OT> The mapper's output data type.
*/
-public class FlatMapDriver<IT, OT> implements PactDriver<GenericFlatMap<IT, OT>, OT> {
+public class FlatMapDriver<IT, OT> implements PactDriver<FlatMapFunction<IT, OT>, OT> {
- private PactTaskContext<GenericFlatMap<IT, OT>, OT> taskContext;
+ private PactTaskContext<FlatMapFunction<IT, OT>, OT> taskContext;
private volatile boolean running;
@Override
- public void setup(PactTaskContext<GenericFlatMap<IT, OT>, OT> context) {
+ public void setup(PactTaskContext<FlatMapFunction<IT, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -55,9 +55,9 @@ public class FlatMapDriver<IT, OT> implements PactDriver<GenericFlatMap<IT, OT>,
}
@Override
- public Class<GenericFlatMap<IT, OT>> getStubType() {
+ public Class<FlatMapFunction<IT, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericFlatMap<IT, OT>> clazz = (Class<GenericFlatMap<IT, OT>>) (Class<?>) GenericFlatMap.class;
+ final Class<FlatMapFunction<IT, OT>> clazz = (Class<FlatMapFunction<IT, OT>>) (Class<?>) FlatMapFunction.class;
return clazz;
}
@@ -75,7 +75,7 @@ public class FlatMapDriver<IT, OT> implements PactDriver<GenericFlatMap<IT, OT>,
public void run() throws Exception {
// cache references on the stack
final MutableObjectIterator<IT> input = this.taskContext.getInput(0);
- final GenericFlatMap<IT, OT> function = this.taskContext.getStub();
+ final FlatMapFunction<IT, OT> function = this.taskContext.getStub();
final Collector<OT> output = this.taskContext.getOutputCollector();
IT record = this.taskContext.<IT>getInputSerializer(0).getSerializer().createInstance();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceCombineDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceCombineDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceCombineDriver.java
index 1d0749c..f786c56 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceCombineDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceCombineDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericCombine;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.runtime.memorymanager.MemoryManager;
@@ -39,12 +39,12 @@ import org.apache.flink.util.MutableObjectIterator;
*
* @param <T> The data type consumed and produced by the combiner.
*/
-public class GroupReduceCombineDriver<T> implements PactDriver<GenericCombine<T>, T> {
+public class GroupReduceCombineDriver<T> implements PactDriver<FlatCombineFunction<T>, T> {
private static final Log LOG = LogFactory.getLog(GroupReduceCombineDriver.class);
- private PactTaskContext<GenericCombine<T>, T> taskContext;
+ private PactTaskContext<FlatCombineFunction<T>, T> taskContext;
private CloseableInputProvider<T> input;
@@ -57,7 +57,7 @@ public class GroupReduceCombineDriver<T> implements PactDriver<GenericCombine<T>
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericCombine<T>, T> context) {
+ public void setup(PactTaskContext<FlatCombineFunction<T>, T> context) {
this.taskContext = context;
this.running = true;
}
@@ -68,9 +68,9 @@ public class GroupReduceCombineDriver<T> implements PactDriver<GenericCombine<T>
}
@Override
- public Class<GenericCombine<T>> getStubType() {
+ public Class<FlatCombineFunction<T>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericCombine<T>> clazz = (Class<GenericCombine<T>>) (Class<?>) GenericCombine.class;
+ final Class<FlatCombineFunction<T>> clazz = (Class<FlatCombineFunction<T>>) (Class<?>) FlatCombineFunction.class;
return clazz;
}
@@ -111,7 +111,7 @@ public class GroupReduceCombineDriver<T> implements PactDriver<GenericCombine<T>
this.serializerFactory.getSerializer(), this.comparator);
// cache references on the stack
- final GenericCombine<T> stub = this.taskContext.getStub();
+ final FlatCombineFunction<T> stub = this.taskContext.getStub();
final Collector<T> output = this.taskContext.getOutputCollector();
// run stub implementation
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
index 1ab080c..960143d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.operators.util.TaskConfig;
@@ -37,13 +37,13 @@ import org.apache.flink.util.MutableObjectIterator;
* The GroupReduceTask creates a iterator over all records from its input. The iterator returns all records grouped by their
* key. The iterator is handed to the <code>reduce()</code> method of the GroupReduceFunction.
*
- * @see GenericGroupReduce
+ * @see org.apache.flink.api.common.functions.GroupReduceFunction
*/
-public class GroupReduceDriver<IT, OT> implements PactDriver<GenericGroupReduce<IT, OT>, OT> {
+public class GroupReduceDriver<IT, OT> implements PactDriver<GroupReduceFunction<IT, OT>, OT> {
private static final Log LOG = LogFactory.getLog(GroupReduceDriver.class);
- private PactTaskContext<GenericGroupReduce<IT, OT>, OT> taskContext;
+ private PactTaskContext<GroupReduceFunction<IT, OT>, OT> taskContext;
private MutableObjectIterator<IT> input;
@@ -56,7 +56,7 @@ public class GroupReduceDriver<IT, OT> implements PactDriver<GenericGroupReduce<
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericGroupReduce<IT, OT>, OT> context) {
+ public void setup(PactTaskContext<GroupReduceFunction<IT, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -67,9 +67,9 @@ public class GroupReduceDriver<IT, OT> implements PactDriver<GenericGroupReduce<
}
@Override
- public Class<GenericGroupReduce<IT, OT>> getStubType() {
+ public Class<GroupReduceFunction<IT, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericGroupReduce<IT, OT>> clazz = (Class<GenericGroupReduce<IT, OT>>) (Class<?>) GenericGroupReduce.class;
+ final Class<GroupReduceFunction<IT, OT>> clazz = (Class<GroupReduceFunction<IT, OT>>) (Class<?>) GroupReduceFunction.class;
return clazz;
}
@@ -100,7 +100,7 @@ public class GroupReduceDriver<IT, OT> implements PactDriver<GenericGroupReduce<
final KeyGroupedIterator<IT> iter = new KeyGroupedIterator<IT>(this.input, this.serializer, this.comparator);
// cache references on the stack
- final GenericGroupReduce<IT, OT> stub = this.taskContext.getStub();
+ final GroupReduceFunction<IT, OT> stub = this.taskContext.getStub();
final Collector<OT> output = this.taskContext.getOutputCollector();
// run stub implementation
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetFirstDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetFirstDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetFirstDriver.java
index b23b0cd..342f307 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetFirstDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetFirstDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypePairComparator;
@@ -32,9 +32,9 @@ import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
-public class JoinWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettablePactDriver<GenericJoiner<IT1, IT2, OT>, OT> {
+public class JoinWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettablePactDriver<FlatJoinFunction<IT1, IT2, OT>, OT> {
- private PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> taskContext;
+ private PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> taskContext;
private CompactingHashTable<IT1> hashTable;
@@ -50,7 +50,7 @@ public class JoinWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettableP
// --------------------------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -61,9 +61,9 @@ public class JoinWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettableP
}
@Override
- public Class<GenericJoiner<IT1, IT2, OT>> getStubType() {
+ public Class<FlatJoinFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericJoiner<IT1, IT2, OT>> clazz = (Class<GenericJoiner<IT1, IT2, OT>>) (Class<?>) GenericJoiner.class;
+ final Class<FlatJoinFunction<IT1, IT2, OT>> clazz = (Class<FlatJoinFunction<IT1, IT2, OT>>) (Class<?>) FlatJoinFunction.class;
return clazz;
}
@@ -126,7 +126,7 @@ public class JoinWithSolutionSetFirstDriver<IT1, IT2, OT> implements ResettableP
@Override
public void run() throws Exception {
- final GenericJoiner<IT1, IT2, OT> joinFunction = taskContext.getStub();
+ final FlatJoinFunction<IT1, IT2, OT> joinFunction = taskContext.getStub();
final Collector<OT> collector = taskContext.getOutputCollector();
IT1 buildSideRecord = this.solutionSideRecord;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetSecondDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetSecondDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetSecondDriver.java
index 4fa5c5a..c38a81a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetSecondDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/JoinWithSolutionSetSecondDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypePairComparator;
@@ -32,9 +32,9 @@ import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
-public class JoinWithSolutionSetSecondDriver<IT1, IT2, OT> implements ResettablePactDriver<GenericJoiner<IT1, IT2, OT>, OT> {
+public class JoinWithSolutionSetSecondDriver<IT1, IT2, OT> implements ResettablePactDriver<FlatJoinFunction<IT1, IT2, OT>, OT> {
- private PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> taskContext;
+ private PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> taskContext;
private CompactingHashTable<IT2> hashTable;
@@ -50,7 +50,7 @@ public class JoinWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resettable
// --------------------------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -61,9 +61,9 @@ public class JoinWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resettable
}
@Override
- public Class<GenericJoiner<IT1, IT2, OT>> getStubType() {
+ public Class<FlatJoinFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericJoiner<IT1, IT2, OT>> clazz = (Class<GenericJoiner<IT1, IT2, OT>>) (Class<?>) GenericJoiner.class;
+ final Class<FlatJoinFunction<IT1, IT2, OT>> clazz = (Class<FlatJoinFunction<IT1, IT2, OT>>) (Class<?>) FlatJoinFunction.class;
return clazz;
}
@@ -126,7 +126,7 @@ public class JoinWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resettable
@Override
public void run() throws Exception {
- final GenericJoiner<IT1, IT2, OT> joinFunction = taskContext.getStub();
+ final FlatJoinFunction<IT1, IT2, OT> joinFunction = taskContext.getStub();
final Collector<OT> collector = taskContext.getOutputCollector();
IT2 buildSideRecord = this.solutionSideRecord;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java
index fe1e0c1..89fbf4d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -31,20 +31,20 @@ import org.apache.flink.util.MutableObjectIterator;
* The MapTask creates an iterator over all key-value pairs of its input and hands that to the <code>map()</code> method
* of the MapFunction.
*
- * @see GenericMap
+ * @see org.apache.flink.api.common.functions.MapFunction
*
* @param <IT> The mapper's input data type.
* @param <OT> The mapper's output data type.
*/
-public class MapDriver<IT, OT> implements PactDriver<GenericMap<IT, OT>, OT> {
+public class MapDriver<IT, OT> implements PactDriver<MapFunction<IT, OT>, OT> {
- private PactTaskContext<GenericMap<IT, OT>, OT> taskContext;
+ private PactTaskContext<MapFunction<IT, OT>, OT> taskContext;
private volatile boolean running;
@Override
- public void setup(PactTaskContext<GenericMap<IT, OT>, OT> context) {
+ public void setup(PactTaskContext<MapFunction<IT, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -55,9 +55,9 @@ public class MapDriver<IT, OT> implements PactDriver<GenericMap<IT, OT>, OT> {
}
@Override
- public Class<GenericMap<IT, OT>> getStubType() {
+ public Class<MapFunction<IT, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericMap<IT, OT>> clazz = (Class<GenericMap<IT, OT>>) (Class<?>) GenericMap.class;
+ final Class<MapFunction<IT, OT>> clazz = (Class<MapFunction<IT, OT>>) (Class<?>) MapFunction.class;
return clazz;
}
@@ -75,7 +75,7 @@ public class MapDriver<IT, OT> implements PactDriver<GenericMap<IT, OT>, OT> {
public void run() throws Exception {
// cache references on the stack
final MutableObjectIterator<IT> input = this.taskContext.getInput(0);
- final GenericMap<IT, OT> function = this.taskContext.getStub();
+ final MapFunction<IT, OT> function = this.taskContext.getStub();
final Collector<OT> output = this.taskContext.getOutputCollector();
IT record = this.taskContext.<IT>getInputSerializer(0).getSerializer().createInstance();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MatchDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MatchDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MatchDriver.java
index a29afa2..e205f1e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MatchDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/MatchDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -42,13 +42,13 @@ import org.apache.flink.util.MutableObjectIterator;
* The MatchTask matches all pairs of records that share the same key and come from different inputs. Each pair of
* matching records is handed to the <code>match()</code> method of the JoinFunction.
*
- * @see GenericJoiner
+ * @see org.apache.flink.api.common.functions.FlatJoinFunction
*/
-public class MatchDriver<IT1, IT2, OT> implements PactDriver<GenericJoiner<IT1, IT2, OT>, OT> {
+public class MatchDriver<IT1, IT2, OT> implements PactDriver<FlatJoinFunction<IT1, IT2, OT>, OT> {
protected static final Log LOG = LogFactory.getLog(MatchDriver.class);
- protected PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> taskContext;
+ protected PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> taskContext;
private volatile JoinTaskIterator<IT1, IT2, OT> matchIterator; // the iterator that does the actual matching
@@ -57,7 +57,7 @@ public class MatchDriver<IT1, IT2, OT> implements PactDriver<GenericJoiner<IT1,
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericJoiner<IT1, IT2, OT>, OT> context) {
+ public void setup(PactTaskContext<FlatJoinFunction<IT1, IT2, OT>, OT> context) {
this.taskContext = context;
this.running = true;
}
@@ -68,9 +68,9 @@ public class MatchDriver<IT1, IT2, OT> implements PactDriver<GenericJoiner<IT1,
}
@Override
- public Class<GenericJoiner<IT1, IT2, OT>> getStubType() {
+ public Class<FlatJoinFunction<IT1, IT2, OT>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericJoiner<IT1, IT2, OT>> clazz = (Class<GenericJoiner<IT1, IT2, OT>>) (Class<?>) GenericJoiner.class;
+ final Class<FlatJoinFunction<IT1, IT2, OT>> clazz = (Class<FlatJoinFunction<IT1, IT2, OT>>) (Class<?>) FlatJoinFunction.class;
return clazz;
}
@@ -141,7 +141,7 @@ public class MatchDriver<IT1, IT2, OT> implements PactDriver<GenericJoiner<IT1,
@Override
public void run() throws Exception {
- final GenericJoiner<IT1, IT2, OT> matchStub = this.taskContext.getStub();
+ final FlatJoinFunction<IT1, IT2, OT> matchStub = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
final JoinTaskIterator<IT1, IT2, OT> matchIterator = this.matchIterator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/NoOpDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/NoOpDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/NoOpDriver.java
index ffe27e6..33d8a18 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/NoOpDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/NoOpDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
@@ -28,15 +28,15 @@ import org.apache.flink.util.MutableObjectIterator;
*
* @param <T> The data type.
*/
-public class NoOpDriver<T> implements PactDriver<AbstractFunction, T> {
+public class NoOpDriver<T> implements PactDriver<AbstractRichFunction, T> {
- private PactTaskContext<AbstractFunction, T> taskContext;
+ private PactTaskContext<AbstractRichFunction, T> taskContext;
private volatile boolean running;
@Override
- public void setup(PactTaskContext<AbstractFunction, T> context) {
+ public void setup(PactTaskContext<AbstractRichFunction, T> context) {
this.taskContext = context;
this.running = true;
}
@@ -47,7 +47,7 @@ public class NoOpDriver<T> implements PactDriver<AbstractFunction, T> {
}
@Override
- public Class<AbstractFunction> getStubType() {
+ public Class<AbstractRichFunction> getStubType() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceCombineDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceCombineDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceCombineDriver.java
index 4d72085..87cea30 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceCombineDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceCombineDriver.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -44,7 +44,7 @@ import org.apache.flink.util.MutableObjectIterator;
*
* @param <T> The data type consumed and produced by the combiner.
*/
-public class ReduceCombineDriver<T> implements PactDriver<GenericReduce<T>, T> {
+public class ReduceCombineDriver<T> implements PactDriver<ReduceFunction<T>, T> {
private static final Log LOG = LogFactory.getLog(ReduceCombineDriver.class);
@@ -52,13 +52,13 @@ public class ReduceCombineDriver<T> implements PactDriver<GenericReduce<T>, T> {
private static final int THRESHOLD_FOR_IN_PLACE_SORTING = 32;
- private PactTaskContext<GenericReduce<T>, T> taskContext;
+ private PactTaskContext<ReduceFunction<T>, T> taskContext;
private TypeSerializer<T> serializer;
private TypeComparator<T> comparator;
- private GenericReduce<T> reducer;
+ private ReduceFunction<T> reducer;
private Collector<T> output;
@@ -75,7 +75,7 @@ public class ReduceCombineDriver<T> implements PactDriver<GenericReduce<T>, T> {
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericReduce<T>, T> context) {
+ public void setup(PactTaskContext<ReduceFunction<T>, T> context) {
this.taskContext = context;
this.running = true;
}
@@ -86,9 +86,9 @@ public class ReduceCombineDriver<T> implements PactDriver<GenericReduce<T>, T> {
}
@Override
- public Class<GenericReduce<T>> getStubType() {
+ public Class<ReduceFunction<T>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericReduce<T>> clazz = (Class<GenericReduce<T>>) (Class<?>) GenericReduce.class;
+ final Class<ReduceFunction<T>> clazz = (Class<ReduceFunction<T>>) (Class<?>) ReduceFunction.class;
return clazz;
}
@@ -168,7 +168,7 @@ public class ReduceCombineDriver<T> implements PactDriver<GenericReduce<T>, T> {
final TypeSerializer<T> serializer = this.serializer;
final TypeComparator<T> comparator = this.comparator;
- final GenericReduce<T> function = this.reducer;
+ final ReduceFunction<T> function = this.reducer;
final Collector<T> output = this.output;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceDriver.java
index a7e9305..9495cdc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/ReduceDriver.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.operators.util.TaskConfig;
@@ -36,13 +36,13 @@ import org.apache.flink.util.MutableObjectIterator;
* The ReduceTask creates a iterator over all records from its input. The iterator returns all records grouped by their
* key. The iterator is handed to the <code>reduce()</code> method of the ReduceFunction.
*
- * @see GenericReduce
+ * @see org.apache.flink.api.common.functions.ReduceFunction
*/
-public class ReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
+public class ReduceDriver<T> implements PactDriver<ReduceFunction<T>, T> {
private static final Log LOG = LogFactory.getLog(ReduceDriver.class);
- private PactTaskContext<GenericReduce<T>, T> taskContext;
+ private PactTaskContext<ReduceFunction<T>, T> taskContext;
private MutableObjectIterator<T> input;
@@ -55,7 +55,7 @@ public class ReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
// ------------------------------------------------------------------------
@Override
- public void setup(PactTaskContext<GenericReduce<T>, T> context) {
+ public void setup(PactTaskContext<ReduceFunction<T>, T> context) {
this.taskContext = context;
this.running = true;
}
@@ -66,9 +66,9 @@ public class ReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
}
@Override
- public Class<GenericReduce<T>> getStubType() {
+ public Class<ReduceFunction<T>> getStubType() {
@SuppressWarnings("unchecked")
- final Class<GenericReduce<T>> clazz = (Class<GenericReduce<T>>) (Class<?>) GenericReduce.class;
+ final Class<ReduceFunction<T>> clazz = (Class<ReduceFunction<T>>) (Class<?>) ReduceFunction.class;
return clazz;
}
@@ -101,7 +101,7 @@ public class ReduceDriver<T> implements PactDriver<GenericReduce<T>, T> {
final TypeSerializer<T> serializer = this.serializer;
final TypeComparator<T> comparator = this.comparator;
- final GenericReduce<T> function = this.taskContext.getStub();
+ final ReduceFunction<T> function = this.taskContext.getStub();
final Collector<T> output = this.taskContext.getOutputCollector();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 45fe05a..c8f217c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -24,8 +24,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GenericCombine;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -497,7 +498,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
if (this.stub != null) {
try {
Configuration stubConfig = this.config.getStubParameters();
- this.stub.open(stubConfig);
+ FunctionUtils.openFunction(this.stub, stubConfig);
stubOpen = true;
}
catch (Throwable t) {
@@ -510,7 +511,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// close. We close here such that a regular close throwing an exception marks a task as failed.
if (this.running && this.stub != null) {
- this.stub.close();
+ FunctionUtils.closeFunction(this.stub);
stubOpen = false;
}
@@ -525,15 +526,17 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// modify accumulators.ll;
if (this.stub != null) {
// collect the counters from the stub
- Map<String, Accumulator<?,?>> accumulators = this.stub.getRuntimeContext().getAllAccumulators();
- RegularPactTask.reportAndClearAccumulators(getEnvironment(), accumulators, this.chainedTasks);
+ if (FunctionUtils.getFunctionRuntimeContext(this.stub, this.runtimeUdfContext) != null) {
+ Map<String, Accumulator<?, ?>> accumulators = FunctionUtils.getFunctionRuntimeContext(this.stub, this.runtimeUdfContext).getAllAccumulators();
+ RegularPactTask.reportAndClearAccumulators(getEnvironment(), accumulators, this.chainedTasks);
+ }
}
}
catch (Exception ex) {
// close the input, but do not report any exceptions, since we already have another root cause
if (stubOpen) {
try {
- this.stub.close();
+ FunctionUtils.closeFunction(this.stub);
}
catch (Throwable t) {}
}
@@ -582,9 +585,12 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// tasks. Type conflicts can occur here if counters with same name but
// different type were used.
+
for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
- Map<String, Accumulator<?, ?>> chainedAccumulators = chainedTask.getStub().getRuntimeContext().getAllAccumulators();
- AccumulatorHelper.mergeInto(accumulators, chainedAccumulators);
+ if (FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null) != null) {
+ Map<String, Accumulator<?, ?>> chainedAccumulators = FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null).getAllAccumulators();
+ AccumulatorHelper.mergeInto(accumulators, chainedAccumulators);
+ }
}
// Don't report if the UDF didn't collect any accumulators
@@ -607,7 +613,9 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// done before sending
AccumulatorHelper.resetAndClearAccumulators(accumulators);
for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
- AccumulatorHelper.resetAndClearAccumulators(chainedTask.getStub().getRuntimeContext().getAllAccumulators());
+ if (FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null) != null) {
+ AccumulatorHelper.resetAndClearAccumulators(FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null).getAllAccumulators());
+ }
}
}
@@ -693,7 +701,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" +
stubSuperClass.getName() + "' as is required.");
}
- stub.setRuntimeContext(this.runtimeUdfContext);
+ FunctionUtils.setFunctionRuntimeContext(stub, this.runtimeUdfContext);
return stub;
}
catch (ClassCastException ccex) {
@@ -988,13 +996,13 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
}
- if (!(localStub instanceof GenericCombine)) {
+ if (!(localStub instanceof FlatCombineFunction)) {
throw new IllegalStateException("Performing combining sort outside a reduce task!");
}
@SuppressWarnings({ "rawtypes", "unchecked" })
CombiningUnilateralSortMerger<?> cSorter = new CombiningUnilateralSortMerger(
- (GenericCombine) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
+ (FlatCombineFunction) localStub, getMemoryManager(), getIOManager(), this.inputIterators[inputNum],
this, this.inputSerializers[inputNum], getLocalStrategyComparator(inputNum),
this.config.getRelativeMemoryInput(inputNum), this.config.getFilehandlesInput(inputNum),
this.config.getSpillingThresholdInput(inputNum));
@@ -1375,7 +1383,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// --------------------------------------------------------------------------------------------
/**
- * Opens the given stub using its {@link Function#open(Configuration)} method. If the open call produces
+ * Opens the given stub using its {@link org.apache.flink.api.common.functions.RichFunction#open(Configuration)} method. If the open call produces
* an exception, a new exception with a standard error message is created, using the encountered exception
* as its cause.
*
@@ -1386,14 +1394,14 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*/
public static void openUserCode(Function stub, Configuration parameters) throws Exception {
try {
- stub.open(parameters);
+ FunctionUtils.openFunction(stub, parameters);
} catch (Throwable t) {
throw new Exception("The user defined 'open(Configuration)' method in " + stub.getClass().toString() + " caused an exception: " + t.getMessage(), t);
}
}
/**
- * Closes the given stub using its {@link Function#close()} method. If the close call produces
+ * Closes the given stub using its {@link org.apache.flink.api.common.functions.RichFunction#close()} method. If the close call produces
* an exception, a new exception with a standard error message is created, using the encountered exception
* as its cause.
*
@@ -1403,7 +1411,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*/
public static void closeUserCode(Function stub) throws Exception {
try {
- stub.close();
+ FunctionUtils.closeFunction(stub);
} catch (Throwable t) {
throw new Exception("The user defined 'close()' method caused an exception: " + t.getMessage(), t);
}
@@ -1505,4 +1513,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
}
return a;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedCollectorMapDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedCollectorMapDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedCollectorMapDriver.java
index 3f7ad61..3dbab78 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedCollectorMapDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedCollectorMapDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators.chaining;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
@@ -61,7 +61,7 @@ public class ChainedCollectorMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
// --------------------------------------------------------------------------------------------
- public Function getStub() {
+ public RichFunction getStub() {
return this.mapper;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedFlatMapDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedFlatMapDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedFlatMapDriver.java
index cca6838..db134a8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedFlatMapDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedFlatMapDriver.java
@@ -19,25 +19,26 @@
package org.apache.flink.runtime.operators.chaining;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.operators.RegularPactTask;
public class ChainedFlatMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
- private GenericFlatMap<IT, OT> mapper;
+ private FlatMapFunction<IT, OT> mapper;
// --------------------------------------------------------------------------------------------
@Override
public void setup(AbstractInvokable parent) {
@SuppressWarnings("unchecked")
- final GenericFlatMap<IT, OT> mapper =
- RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, GenericFlatMap.class);
+ final FlatMapFunction<IT, OT> mapper =
+ RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, FlatMapFunction.class);
this.mapper = mapper;
- mapper.setRuntimeContext(getUdfRuntimeContext());
+ FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
@Override
@@ -54,8 +55,9 @@ public class ChainedFlatMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
@Override
public void cancelTask() {
try {
- this.mapper.close();
- } catch (Throwable t) {
+ FunctionUtils.closeFunction(this.mapper);
+ }
+ catch (Throwable t) {
}
}
@@ -84,4 +86,5 @@ public class ChainedFlatMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
public void close() {
this.outputCollector.close();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java
index 3ae324c..a7b1048 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java
@@ -20,24 +20,25 @@
package org.apache.flink.runtime.operators.chaining;
import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.operators.RegularPactTask;
public class ChainedMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
- private GenericMap<IT, OT> mapper;
+ private MapFunction<IT, OT> mapper;
// --------------------------------------------------------------------------------------------
@Override
public void setup(AbstractInvokable parent) {
@SuppressWarnings("unchecked")
- final GenericMap<IT, OT> mapper =
- RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, GenericMap.class);
+ final MapFunction<IT, OT> mapper =
+ RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
this.mapper = mapper;
- mapper.setRuntimeContext(getUdfRuntimeContext());
+ FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
@Override
@@ -54,7 +55,7 @@ public class ChainedMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
@Override
public void cancelTask() {
try {
- this.mapper.close();
+ FunctionUtils.closeFunction(this.mapper);
} catch (Throwable t) {
}
}
@@ -84,4 +85,5 @@ public class ChainedMapDriver<IT, OT> extends ChainedDriver<IT, OT> {
public void close() {
this.outputCollector.close();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedTerminationCriterionDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedTerminationCriterionDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedTerminationCriterionDriver.java
index 76d860b..3a42469 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedTerminationCriterionDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedTerminationCriterionDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators.chaining;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.IterationRuntimeContext;
import org.apache.flink.api.common.operators.base.BulkIterationBase;
import org.apache.flink.api.common.operators.base.BulkIterationBase.TerminationCriterionAggregator;
@@ -47,7 +47,7 @@ public class ChainedTerminationCriterionDriver<IT, OT> extends ChainedDriver<IT,
// --------------------------------------------------------------------------------------------
- public Function getStub() {
+ public RichFunction getStub() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/SynchronousChainedCombineDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/SynchronousChainedCombineDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/SynchronousChainedCombineDriver.java
index d5ce0a7..ffac151 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/SynchronousChainedCombineDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/SynchronousChainedCombineDriver.java
@@ -22,8 +22,9 @@ package org.apache.flink.runtime.operators.chaining;
import java.io.IOException;
import java.util.List;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GenericCombine;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -51,7 +52,7 @@ public class SynchronousChainedCombineDriver<T> extends ChainedDriver<T, T> {
private InMemorySorter<T> sorter;
- private GenericCombine<T> combiner;
+ private FlatCombineFunction<T> combiner;
private TypeSerializer<T> serializer;
@@ -72,10 +73,10 @@ public class SynchronousChainedCombineDriver<T> extends ChainedDriver<T, T> {
this.parent = parent;
@SuppressWarnings("unchecked")
- final GenericCombine<T> combiner =
- RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, GenericCombine.class);
+ final FlatCombineFunction<T> combiner =
+ RegularPactTask.instantiateUserCode(this.config, userCodeClassLoader, FlatCombineFunction.class);
this.combiner = combiner;
- combiner.setRuntimeContext(getUdfRuntimeContext());
+ FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
@Override
@@ -185,7 +186,7 @@ public class SynchronousChainedCombineDriver<T> extends ChainedDriver<T, T> {
this.comparator);
// cache references on the stack
- final GenericCombine<T> stub = this.combiner;
+ final FlatCombineFunction<T> stub = this.combiner;
final Collector<T> output = this.outputCollector;
// run stub implementation
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildFirstHashMatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildFirstHashMatchIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildFirstHashMatchIterator.java
index 65af050..0daf69b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildFirstHashMatchIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildFirstHashMatchIterator.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.hash;
import java.io.IOException;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -101,7 +101,7 @@ public class BuildFirstHashMatchIterator<V1, V2, O> implements JoinTaskIterator<
}
@Override
- public final boolean callWithNextKey(GenericJoiner<V1, V2, O> matchFunction, Collector<O> collector)
+ public final boolean callWithNextKey(FlatJoinFunction<V1, V2, O> matchFunction, Collector<O> collector)
throws Exception
{
if (this.hashJoin.nextRecord())
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildSecondHashMatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildSecondHashMatchIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildSecondHashMatchIterator.java
index e124201..70e5afb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildSecondHashMatchIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/hash/BuildSecondHashMatchIterator.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.hash;
import java.io.IOException;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -100,7 +100,7 @@ public class BuildSecondHashMatchIterator<V1, V2, O> implements JoinTaskIterator
}
@Override
- public boolean callWithNextKey(GenericJoiner<V1, V2, O> matchFunction, Collector<O> collector)
+ public boolean callWithNextKey(FlatJoinFunction<V1, V2, O> matchFunction, Collector<O> collector)
throws Exception
{
if (this.hashJoin.nextRecord())
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
index 06d0ac7..da8a11b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
@@ -29,7 +29,8 @@ import java.util.Queue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericCombine;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -71,7 +72,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
*/
private static final Log LOG = LogFactory.getLog(CombiningUnilateralSortMerger.class);
- private final GenericCombine<E> combineStub; // the user code stub that does the combining
+ private final FlatCombineFunction<E> combineStub; // the user code stub that does the combining
private Configuration udfConfig;
@@ -101,7 +102,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
* @throws MemoryAllocationException Thrown, if not enough memory can be obtained from the memory manager to
* perform the sort.
*/
- public CombiningUnilateralSortMerger(GenericCombine<E> combineStub, MemoryManager memoryManager, IOManager ioManager,
+ public CombiningUnilateralSortMerger(FlatCombineFunction<E> combineStub, MemoryManager memoryManager, IOManager ioManager,
MutableObjectIterator<E> input, AbstractInvokable parentTask,
TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
double memoryFraction, int maxNumFileHandles, float startSpillingFraction)
@@ -133,7 +134,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
* @throws MemoryAllocationException Thrown, if not enough memory can be obtained from the memory manager to
* perform the sort.
*/
- public CombiningUnilateralSortMerger(GenericCombine<E> combineStub, MemoryManager memoryManager, IOManager ioManager,
+ public CombiningUnilateralSortMerger(FlatCombineFunction<E> combineStub, MemoryManager memoryManager, IOManager ioManager,
MutableObjectIterator<E> input, AbstractInvokable parentTask,
TypeSerializerFactory<E> serializerFactory, TypeComparator<E> comparator,
double memoryFraction, int numSortBuffers, int maxNumFileHandles,
@@ -254,12 +255,12 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
// ------------------- Spilling Phase ------------------------
- final GenericCombine<E> combineStub = CombiningUnilateralSortMerger.this.combineStub;
+ final FlatCombineFunction<E> combineStub = CombiningUnilateralSortMerger.this.combineStub;
// now that we are actually spilling, take the combiner, and open it
try {
- Configuration conf = CombiningUnilateralSortMerger.this.udfConfig;
- combineStub.open(conf == null ? new Configuration() : conf);
+ Configuration conf = CombiningUnilateralSortMerger.this.udfConfig;
+ FunctionUtils.openFunction (combineStub, (conf == null ? new Configuration() : conf));
}
catch (Throwable t) {
throw new IOException("The user-defined combiner failed in its 'open()' method.", t);
@@ -380,7 +381,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
// close the user code
try {
- combineStub.close();
+ FunctionUtils.closeFunction(combineStub);
}
catch (Throwable t) {
throw new IOException("The user-defined combiner failed in its 'close()' method.", t);
@@ -466,7 +467,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
this.memManager.getPageSize());
final WriterCollector<E> collector = new WriterCollector<E>(output, this.serializer);
- final GenericCombine<E> combineStub = CombiningUnilateralSortMerger.this.combineStub;
+ final FlatCombineFunction<E> combineStub = CombiningUnilateralSortMerger.this.combineStub;
// combine and write to disk
try {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
index 308e333..2ed75ae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -151,7 +151,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
* @see org.apache.flink.runtime.operators.util.JoinTaskIterator#callWithNextKey()
*/
@Override
- public boolean callWithNextKey(final GenericJoiner<T1, T2, O> matchFunction, final Collector<O> collector)
+ public boolean callWithNextKey(final FlatJoinFunction<T1, T2, O> matchFunction, final Collector<O> collector)
throws Exception
{
if (!this.iterator1.nextKey() || !this.iterator2.nextKey()) {
@@ -234,7 +234,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
* @throws Exception Forwards all exceptions thrown by the stub.
*/
private void crossFirst1withNValues(final T1 val1, final T2 firstValN,
- final Iterator<T2> valsN, final GenericJoiner<T1, T2, O> matchFunction, final Collector<O> collector)
+ final Iterator<T2> valsN, final FlatJoinFunction<T1, T2, O> matchFunction, final Collector<O> collector)
throws Exception
{
this.copy1 = this.serializer1.copy(val1, this.copy1);
@@ -267,7 +267,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
* @throws Exception Forwards all exceptions thrown by the stub.
*/
private void crossSecond1withNValues(T2 val1, T1 firstValN,
- Iterator<T1> valsN, GenericJoiner<T1, T2, O> matchFunction, Collector<O> collector)
+ Iterator<T1> valsN, FlatJoinFunction<T1, T2, O> matchFunction, Collector<O> collector)
throws Exception
{
this.copy2 = this.serializer2.copy(val1, this.copy2);
@@ -280,7 +280,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
if (valsN.hasNext()) {
this.copy2 = this.serializer2.copy(val1, this.copy2);
- matchFunction.join(nRec, this.copy2, collector);
+ matchFunction.join(nRec,this.copy2,collector);
} else {
matchFunction.join(nRec, val1, collector);
more = false;
@@ -297,7 +297,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
*/
private void crossMwithNValues(final T1 firstV1, Iterator<T1> spillVals,
final T2 firstV2, final Iterator<T2> blockVals,
- final GenericJoiner<T1, T2, O> matchFunction, final Collector<O> collector)
+ final FlatJoinFunction<T1, T2, O> matchFunction, final Collector<O> collector)
throws Exception
{
// ==================================================
@@ -411,7 +411,7 @@ public class MergeMatchIterator<T1, T2, O> implements JoinTaskIterator<T1, T2, O
// get instances of key and block value
final T2 nextBlockVal = this.blockIt.next();
this.copy1 = this.serializer1.copy(nextSpillVal, this.copy1);
- matchFunction.join(this.copy1, nextBlockVal, collector);
+ matchFunction.join(this.copy1, nextBlockVal, collector);
}
// reset block iterator
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/JoinTaskIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/JoinTaskIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/JoinTaskIterator.java
index 6bcb6ee..3ed647d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/JoinTaskIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/JoinTaskIterator.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.util;
import java.io.IOException;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.runtime.memorymanager.MemoryAllocationException;
import org.apache.flink.util.Collector;
@@ -60,7 +60,7 @@ public interface JoinTaskIterator<V1, V2, O>
* @return True, if a next key exists, false if no more keys exist.
* @throws Exception Exceptions from the user code are forwarded.
*/
- boolean callWithNextKey(GenericJoiner<V1, V2, O> matchFunction, Collector<O> collector) throws Exception;
+ boolean callWithNextKey(FlatJoinFunction<V1, V2, O> matchFunction, Collector<O> collector) throws Exception;
/**
* Aborts the matching process. This extra abort method is supplied, because a significant time may pass while
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
index e91e100..3894233 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
@@ -23,13 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.java.record.functions.JoinFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
-import org.apache.flink.runtime.operators.BuildFirstCachedMatchDriver;
-import org.apache.flink.runtime.operators.BuildSecondCachedMatchDriver;
-import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
@@ -43,7 +40,7 @@ import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
-public class CachedMatchTaskTest extends DriverTestBase<GenericJoiner<Record, Record, Record>>
+public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record, Record, Record>>
{
private static final long HASH_MEM = 6*1024*1024;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
index 46717be..5551485 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
@@ -23,12 +23,9 @@ import java.util.Iterator;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
-import org.apache.flink.api.java.record.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
-import org.apache.flink.runtime.operators.CoGroupDriver;
-import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
import org.apache.flink.types.IntValue;
@@ -37,7 +34,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class CoGroupTaskExternalITCase extends DriverTestBase<GenericCoGrouper<Record, Record, Record>>
+public class CoGroupTaskExternalITCase extends DriverTestBase<CoGroupFunction<Record, Record, Record>>
{
private static final long SORT_MEM = 3*1024*1024;
@@ -87,7 +84,7 @@ public class CoGroupTaskExternalITCase extends DriverTestBase<GenericCoGrouper<R
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
- public static final class MockCoGroupStub extends CoGroupFunction {
+ public static final class MockCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
private static final long serialVersionUID = 1L;
private final Record res = new Record();
[20/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
new file mode 100644
index 0000000..a4e1248
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCrossFunction.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.CrossFunction;
+
+
+/**
+ * The abstract base class for Cross functions. Cross functions build a Cartesian produce of their inputs
+ * and call the function or each pair of elements.
+ * They are a means of convenience and can be used to directly produce manipulate the
+ * pair of elements, instead of having the operator build 2-tuples, and then using a
+ * MapFunction over those 2-tuples.
+ * <p>
+ * The basic syntax for using Cross on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.cross(set2).with(new MyCrossFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * <p>
+ * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
+public abstract class RichCrossFunction<IN1, IN2, OUT> extends AbstractRichFunction implements CrossFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+
+ /**
+ * The core method of the cross operation. The method will be invoked for each pair of elements
+ * in the Cartesian product.
+ *
+ * @param first The element from the first input.
+ * @param second The element from the second input.
+ * @return The result element.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract OUT cross(IN1 first, IN2 second) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
new file mode 100644
index 0000000..e3baa74
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFilterFunction.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FilterFunction;
+
+/**
+ * The abstract base class for Filter functions. A filter function take elements and evaluates a
+ * predicate on them to decide whether to keep the element, or to discard it.
+ * <p>
+ * The basic syntax for using a FilterFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.filter(new MyFilterFunction());
+ * </blockquote></pre>
+ * <p>
+ * Like all functions, the FilterFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <T> The type of the filtered elements.
+ */
+public abstract class RichFilterFunction<T> extends AbstractRichFunction implements FilterFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The core method of the FilterFunction. The method is called for each element in the input,
+ * and determines whether the element should be kept or filtered out. If the method returns true,
+ * the element passes the filter and is kept, if the method returns false, the element is
+ * filtered out.
+ *
+ * @param value The input value to be filtered.
+ * @return Flag to indicate whether to keep the value (true) or to discard it (false).
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract boolean filter(T value) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
new file mode 100644
index 0000000..8c326c6
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.util.Collector;
+
+import java.util.Iterator;
+
+public abstract class RichFlatCombineFunction<T> extends AbstractRichFunction implements FlatCombineFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public abstract void combine(Iterator<T> values, Collector<T> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
new file mode 100644
index 0000000..15b4539
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatJoinFunction.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.util.Collector;
+
+/**
+ * The abstract base class for Join functions. Join functions combine two data sets by joining their
+ * elements on specified keys and calling this function with each pair of joining elements.
+ * By default, this follows strictly the semantics of an "inner join" in SQL.
+ * the semantics are those of an "inner join", meaning that elements are filtered out
+ * if their key is not contained in the other data set.
+ * <p>
+ * Per the semantics of an inner join, the function is
+ * <p>
+ * The basic syntax for using Join on two data sets is as follows:
+ * <pre><blockquote>
+ * DataSet<X> set1 = ...;
+ * DataSet<Y> set2 = ...;
+ *
+ * set1.join(set2).where(<key-definition>).equalTo(<key-definition>).with(new MyJoinFunction());
+ * </blockquote></pre>
+ * <p>
+ * {@code set1} is here considered the first input, {@code set2} the second input.
+ * The keys can be defined through tuple field positions or key extractors.
+ * See {@link org.apache.flink.api.java.operators.Keys} for details.
+ * <p>
+ * The Join function is actually not a necessary part of a join operation. If no JoinFunction is provided,
+ * the result of the operation is a sequence of Tuple2, where the elements in the tuple are those that
+ * the JoinFunction would have been invoked with.
+ * <P>
+ * Note: You can use a {@link RichCoGroupFunction} to perform an outer join.
+ * <p>
+ * All functions need to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
+public abstract class RichFlatJoinFunction<IN1, IN2, OUT> extends AbstractRichFunction implements FlatJoinFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The user-defined method for performing transformations after a join.
+ * The method is called with matching pairs of elements from the inputs.
+ *
+ * @param first The element from first input.
+ * @param second The element from second input.
+ * @return The resulting element.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract void join(IN1 first, IN2 second, Collector<OUT> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
new file mode 100644
index 0000000..2293b5e
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+/**
+ * The abstract base class for flatMap functions. FlatMap functions take elements and transform them,
+ * into zero, one, or more elements. Typical applications can be splitting elements, or unnesting lists
+ * and arrays. Operations that produce multiple strictly one result element per input element can also
+ * use the {@link RichMapFunction}.
+ * <p>
+ * The basic syntax for using a FlatMapFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<Y> result = input.flatMap(new MyFlatMapFunction());
+ * </blockquote></pre>
+ * <p>
+ * Like all functions, the FlatMapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN> Type of the input elements.
+ * @param <OUT> Type of the returned elements.
+ */
+public abstract class RichFlatMapFunction<IN, OUT> extends AbstractRichFunction implements FlatMapFunction<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The core method of the FlatMapFunction. Takes an element from the input data set and transforms
+ * it into zero, one, or more elements.
+ *
+ * @param value The input value.
+ * @param out The collector for for emitting result values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract void flatMap(IN value, Collector<OUT> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
new file mode 100644
index 0000000..eb75f53
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Iterator;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.util.Collector;
+
+/**
+ * The abstract base class for group reduce functions. Group reduce functions process groups of elements.
+ * They may aggregate them to a single value, or produce multiple result values for each group.
+ * <p>
+ * For a reduce functions that works incrementally by combining always two elements, see
+ * {@link RichReduceFunction}, called via {@link org.apache.flink.api.java.DataSet#reduce(RichReduceFunction)}.
+ * <p>
+ * The basic syntax for using a grouped GroupReduceFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.groupBy(<key-definition>).reduceGroup(new MyGroupReduceFunction());
+ * </blockquote></pre>
+ * <p>
+ * GroupReduceFunctions may be "combinable", in which case they can pre-reduce partial groups in order to
+ * reduce the data volume early. See the {@link #combine(Iterator, Collector)} function for details.
+ * <p>
+ * Like all functions, the GroupReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN> Type of the elements that this function processes.
+ * @param <OUT> The type of the elements returned by the user-defined function.
+ */
+public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunction implements GroupReduceFunction<IN, OUT>, FlatCombineFunction<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Core method of the reduce function. It is called one per group of elements. If the reducer
+ * is not grouped, than the entire data set is considered one group.
+ *
+ * @param values The iterator returning the group of values to be reduced.
+ * @param out The collector to emit the returned values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract void reduce(Iterator<IN> values, Collector<OUT> out) throws Exception;
+
+ /**
+ * The combine methods pre-reduces elements. It may be called on subsets of the data
+ * before the actual reduce function. This is often helpful to lower data volume prior
+ * to reorganizing the data in an expensive way, as might be required for the final
+ * reduce function.
+ * <p>
+ * This method is only ever invoked when the subclass of {@link RichGroupReduceFunction}
+ * adds the {@link Combinable} annotation, or if the <i>combinable</i> flag is set when defining
+ * the <i>reduceGroup<i> operation via
+ * {@link org.apache.flink.api.java.operators.GroupReduceOperator#setCombinable(boolean)}.
+ * <p>
+ * Since the reduce function will be called on the result of this method, it is important that this
+ * method returns the same data type as it consumes. By default, this method only calls the
+ * {@link #reduce(Iterator, Collector)} method. If the behavior in the pre-reducing is different
+ * from the final reduce function (for example because the reduce function changes the data type),
+ * this method must be overwritten, or the execution will fail.
+ *
+ * @param values The iterator returning the group of values to be reduced.
+ * @param out The collector to emit the returned values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public void combine(Iterator<IN> values, Collector<IN> out) throws Exception {
+ @SuppressWarnings("unchecked")
+ Collector<OUT> c = (Collector<OUT>) out;
+ reduce(values, c);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * This annotation can be added to classes that extend {@link RichGroupReduceFunction}, in oder to mark
+ * them as "combinable". The system may call the {@link RichGroupReduceFunction#combine(Iterator, Collector)}
+ * method on such functions, to pre-reduce the data before transferring it over the network to
+ * the actual group reduce operation.
+ * <p>
+ * Marking combinable functions as such is in general beneficial for performance.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.TYPE)
+ public static @interface Combinable {};
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
new file mode 100644
index 0000000..7eaf44c
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+
+public abstract class RichJoinFunction<IN1,IN2,OUT> extends AbstractRichFunction implements JoinFunction<IN1,IN2,OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public abstract OUT join(IN1 first, IN2 second) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
new file mode 100644
index 0000000..54de7d4
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+
+/**
+ * The abstract base class for Map functions. Map functions take elements and transform them,
+ * element wise. A Map function always produces a single result element for each input element.
+ * Typical applications are parsing elements, converting data types, or projecting out fields.
+ * Operations that produce multiple result elements from a single input element can be implemented
+ * using the {@link RichFlatMapFunction}.
+ * <p>
+ * The basic syntax for using a MapFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<Y> result = input.map(new MyMapFunction());
+ * </blockquote></pre>
+ * <p>
+ * Like all functions, the MapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <IN> Type of the input elements.
+ * @param <OUT> Type of the returned elements.
+ */
+public abstract class RichMapFunction<IN, OUT> extends AbstractRichFunction implements MapFunction<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The core method of the MapFunction. Takes an element from the input data set and transforms
+ * it into another element.
+ *
+ * @param value The input value.
+ * @return The value produced by the map function from the input value.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ @Override
+ public abstract OUT map(IN value) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
new file mode 100644
index 0000000..35cb392
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+
+/**
+ * The abstract base class for Reduce functions. Reduce functions combine groups of elements to
+ * a single value, by taking always two elements and combining them into one. Reduce functions
+ * may be used on entire data sets, or on grouped data sets. In the latter case, each group is reduced
+ * individually.
+ * <p>
+ * For a reduce functions that work on an entire group at the same time (such as the
+ * MapReduce/Hadoop-style reduce), see {@link RichGroupReduceFunction}, called via
+ * {@link org.apache.flink.api.java.DataSet#reduceGroup(RichGroupReduceFunction)}. In the general case,
+ * ReduceFunctions are considered faster, because they allow the system to use hash-based
+ * execution strategies.
+ * <p>
+ * The basic syntax for using a grouped ReduceFunction is as follows:
+ * <pre><blockquote>
+ * DataSet<X> input = ...;
+ *
+ * DataSet<X> result = input.groupBy(<key-definition>).reduce(new MyReduceFunction());
+ * </blockquote></pre>
+ * <p>
+ * Like all functions, the ReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ *
+ * @param <T> Type of the elements that this function processes.
+ */
+public abstract class RichReduceFunction<T> extends AbstractRichFunction implements ReduceFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The core method of the ReduceFunction, combining two values into one value of the same type.
+ * The reduce function is consecutively applied to all values of a group until only a single value remains.
+ *
+ * @param value1 The first value to combine.
+ * @param value2 The second value to combine.
+ * @return The combined value of both input values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ public abstract T reduce(T value1, T value2) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/functions/UnsupportedLambdaExpressionException.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/UnsupportedLambdaExpressionException.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/UnsupportedLambdaExpressionException.java
new file mode 100644
index 0000000..ccc4685
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/UnsupportedLambdaExpressionException.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.functions;
+
+import org.apache.flink.api.common.InvalidProgramException;
+
+public class UnsupportedLambdaExpressionException extends InvalidProgramException {
+
+ private static final long serialVersionUID = -1721898801986321010L;
+
+ public UnsupportedLambdaExpressionException() {
+ super("Java 8 lambda expressions are currently supported only in filter and reduce user-defined functions.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
index ca6ed94..80a5fa0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -32,8 +32,8 @@ import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.java.aggregation.AggregationFunction;
import org.apache.flink.api.java.aggregation.AggregationFunctionFactory;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction.Combinable;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.configuration.Configuration;
@@ -151,7 +151,7 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
@SuppressWarnings("unchecked")
@Override
- protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>> translateToDataFlow(Operator<IN> input) {
+ protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> translateToDataFlow(Operator<IN> input) {
// sanity check
if (this.aggregationFunctions.isEmpty() || this.aggregationFunctions.size() != this.fields.size()) {
@@ -174,7 +174,7 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
@SuppressWarnings("rawtypes")
- GroupReduceFunction<IN, IN> function = new AggregatingUdf(aggFunctions, fields);
+ RichGroupReduceFunction<IN, IN> function = new AggregatingUdf(aggFunctions, fields);
String name = getName() != null ? getName() : genName.toString();
@@ -183,8 +183,8 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
if (this.grouping == null) {
// non grouped aggregation
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>> po =
- new GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>>(function, operatorInfo, new int[0], name);
+ GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po =
+ new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, new int[0], name);
po.setCombinable(true);
@@ -200,8 +200,8 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
// grouped aggregation
int[] logicalKeyPositions = this.grouping.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>> po =
- new GroupReduceOperatorBase<IN, IN, GenericGroupReduce<IN, IN>>(function, operatorInfo, logicalKeyPositions, name);
+ GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>> po =
+ new GroupReduceOperatorBase<IN, IN, GroupReduceFunction<IN, IN>>(function, operatorInfo, logicalKeyPositions, name);
po.setCombinable(true);
@@ -245,7 +245,7 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
// --------------------------------------------------------------------------------------------
@Combinable
- public static final class AggregatingUdf<T extends Tuple> extends GroupReduceFunction<T, T> {
+ public static final class AggregatingUdf<T extends Tuple> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
private final int[] fieldPositions;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
index 8748556..89c3334 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
@@ -21,8 +21,9 @@ package org.apache.flink.api.java.operators;
import java.security.InvalidParameterException;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -30,8 +31,8 @@ import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration.SolutionSetPlaceHolder;
-import org.apache.flink.api.java.functions.CoGroupFunction;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
import org.apache.flink.api.java.operators.Keys.FieldPositionKeys;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingCoGroupOperator;
@@ -123,8 +124,8 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
- CoGroupOperatorBase<I1, I2, OUT, GenericCoGrouper<I1, I2, OUT>> po =
- new CoGroupOperatorBase<I1, I2, OUT, GenericCoGrouper<I1, I2, OUT>>(
+ CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>> po =
+ new CoGroupOperatorBase<I1, I2, OUT, CoGroupFunction<I1, I2, OUT>>(
function, new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()),
logicalKeyPositions1, logicalKeyPositions2, name);
@@ -199,10 +200,10 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
final KeyExtractingMapper<I1, K> extractor1 = new KeyExtractingMapper<I1, K>(keys1.getKeyExtractor());
final KeyExtractingMapper<I2, K> extractor2 = new KeyExtractingMapper<I2, K>(keys2.getKeyExtractor());
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup = new PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
cogroup.setFirstInput(keyMapper1);
@@ -236,10 +237,10 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
final TupleKeyExtractingMapper<I1, K> extractor1 = new TupleKeyExtractingMapper<I1, K>(logicalKeyPositions1[0]);
final KeyExtractingMapper<I2, K> extractor2 = new KeyExtractingMapper<I2, K>(keys2.getKeyExtractor());
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup = new PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>(function, logicalKeyPositions1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
@@ -274,10 +275,10 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
final KeyExtractingMapper<I1, K> extractor1 = new KeyExtractingMapper<I1, K>(keys1.getKeyExtractor());
final TupleKeyExtractingMapper<I2, K> extractor2 = new TupleKeyExtractingMapper<I2, K>(logicalKeyPositions2[0]);
- final MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>> keyMapper1 =
- new MapOperatorBase<I1, Tuple2<K, I1>, GenericMap<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
- final MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>> keyMapper2 =
- new MapOperatorBase<I2, Tuple2<K, I2>, GenericMap<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
+ final MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>> keyMapper1 =
+ new MapOperatorBase<I1, Tuple2<K, I1>, MapFunction<I1, Tuple2<K, I1>>>(extractor1, new UnaryOperatorInformation<I1, Tuple2<K, I1>>(inputType1, typeInfoWithKey1), "Key Extractor 1");
+ final MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>> keyMapper2 =
+ new MapOperatorBase<I2, Tuple2<K, I2>, MapFunction<I2, Tuple2<K, I2>>>(extractor2, new UnaryOperatorInformation<I2, Tuple2<K, I2>>(inputType2, typeInfoWithKey2), "Key Extractor 2");
final PlanUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup = new PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>(function, keys1, logicalKeyPositions2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);
@@ -407,7 +408,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @param field0 The first index of the Tuple fields of the second co-grouped DataSets that should be used as key
* @param fields The indexes of the Tuple fields of the second co-grouped DataSet that should be used as keys.
* @return An incomplete CoGroup transformation.
- * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(CoGroupFunction)} to finalize the CoGroup transformation.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.java.functions.RichCoGroupFunction)} to finalize the CoGroup transformation.
*/
public CoGroupOperatorWithoutFunction equalTo(int field0, int... fields) {
int[] actualFields = new int[fields.length + 1];
@@ -423,7 +424,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @param field0 The first field of the second co-grouped DataSets that should be used as key
* @param fields The fields of the first co-grouped DataSets that should be used as keys.
* @return An incomplete CoGroup transformation.
- * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(CoGroupFunction)} to finalize the CoGroup transformation.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.java.functions.RichCoGroupFunction)} to finalize the CoGroup transformation.
*/
public CoGroupOperatorWithoutFunction equalTo(String field0, String... fields) {
String[] actualFields = new String[fields.length + 1];
@@ -439,7 +440,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
*
* @param keyExtractor The KeySelector function which extracts the key values from the second DataSet on which it is grouped.
* @return An incomplete CoGroup transformation.
- * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(CoGroupFunction)} to finalize the CoGroup transformation.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.java.functions.RichCoGroupFunction)} to finalize the CoGroup transformation.
*/
public <K> CoGroupOperatorWithoutFunction equalTo(KeySelector<I2, K> keyExtractor) {
return createCoGroupOperator(new Keys.SelectorFunctionKeys<I2, K>(keyExtractor, input2.getType()));
@@ -447,8 +448,8 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
/**
* Intermediate step of a CoGroup transformation. <br/>
- * To continue the CoGroup transformation, provide a {@link CoGroupFunction} by calling
- * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(CoGroupFunction)}.
+ * To continue the CoGroup transformation, provide a {@link org.apache.flink.api.java.functions.RichCoGroupFunction} by calling
+ * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.java.functions.RichCoGroupFunction)}.
*
*/
private CoGroupOperatorWithoutFunction createCoGroupOperator(Keys<I2> keys2) {
@@ -501,19 +502,22 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
}
/**
- * Finalizes a CoGroup transformation by applying a {@link CoGroupFunction} to groups of elements with identical keys.<br/>
+ * Finalizes a CoGroup transformation by applying a {@link org.apache.flink.api.java.functions.RichCoGroupFunction} to groups of elements with identical keys.<br/>
* Each CoGroupFunction call returns an arbitrary number of keys.
*
* @param function The CoGroupFunction that is called for all groups of elements with identical keys.
* @return An CoGroupOperator that represents the co-grouped result DataSet.
*
- * @see CoGroupFunction
+ * @see org.apache.flink.api.java.functions.RichCoGroupFunction
* @see DataSet
*/
public <R> CoGroupOperator<I1, I2, R> with(CoGroupFunction<I1, I2, R> function) {
if (function == null) {
throw new NullPointerException("CoGroup function must not be null.");
}
+ if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
TypeInformation<R> returnType = TypeExtractor.getCoGroupReturnTypes(function, input1.getType(), input2.getType());
return new CoGroupOperator<I1, I2, R>(input1, input2, keys1, keys2, function, returnType);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
index 036d292..d1e99d6 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
@@ -21,14 +21,15 @@ package org.apache.flink.api.java.operators;
import java.util.Arrays;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.GenericCrosser;
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputSemanticProperties;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.base.CrossOperatorBase;
import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.functions.CrossFunction;
import org.apache.flink.api.java.functions.SemanticPropUtil;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.types.TypeInformation;
@@ -71,12 +72,12 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
}
@Override
- protected org.apache.flink.api.common.operators.base.CrossOperatorBase<I1, I2, OUT, GenericCrosser<I1,I2,OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
+ protected org.apache.flink.api.common.operators.base.CrossOperatorBase<I1, I2, OUT, CrossFunction<I1,I2,OUT>> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
String name = getName() != null ? getName() : function.getClass().getName();
// create operator
- CrossOperatorBase<I1, I2, OUT, GenericCrosser<I1, I2, OUT>> po =
- new CrossOperatorBase<I1, I2, OUT, GenericCrosser<I1, I2, OUT>>(function, new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()), name);
+ CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>> po =
+ new CrossOperatorBase<I1, I2, OUT, CrossFunction<I1, I2, OUT>>(function, new BinaryOperatorInformation<I1, I2, OUT>(getInput1Type(), getInput2Type(), getResultType()), name);
// set inputs
po.setFirstInput(input1);
po.setSecondInput(input2);
@@ -133,6 +134,9 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
if (function == null) {
throw new NullPointerException("Cross function must not be null.");
}
+ if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, input1.getType(), input2.getType());
return new CrossOperator<I1, I2, R>(input1, input2, function, returnType);
}
@@ -220,7 +224,7 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
}
}
- public static final class ProjectCrossFunction<T1, T2, R extends Tuple> extends CrossFunction<T1, T2, R> {
+ public static final class ProjectCrossFunction<T1, T2, R extends Tuple> implements CrossFunction<T1, T2, R> {
private static final long serialVersionUID = 1L;
@@ -1398,7 +1402,7 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
// default join functions
// --------------------------------------------------------------------------------------------
- public static final class DefaultCrossFunction<T1, T2> extends CrossFunction<T1, T2, Tuple2<T1, T2>> {
+ public static final class DefaultCrossFunction<T1, T2> implements CrossFunction<T1, T2, Tuple2<T1, T2>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
index cb7db06..591551f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
@@ -21,13 +21,14 @@ package org.apache.flink.api.java.operators;
import java.util.Iterator;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -78,15 +79,17 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
@Override
protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<?, T, ?> translateToDataFlow(Operator<T> input) {
- GroupReduceFunction<T, T> function = new DistinctFunction<T>();
+ final RichGroupReduceFunction<T, T> function = new DistinctFunction<T>();
+ final FlatCombineFunction<T> combineFunction = new DistinctCombiner<T>();
+
String name = function.getClass().getName();
if (keys instanceof Keys.FieldPositionKeys) {
int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<T, T>(getInputType(), getResultType());
- GroupReduceOperatorBase<T, T, GenericGroupReduce<T, T>> po =
- new GroupReduceOperatorBase<T, T, GenericGroupReduce<T, T>>(function, operatorInfo, logicalKeyPositions, name);
+ GroupReduceOperatorBase<T, T, GroupReduceFunction<T, T>> po =
+ new GroupReduceOperatorBase<T, T, GroupReduceFunction<T, T>>(function, operatorInfo, logicalKeyPositions, name);
po.setCombinable(true);
po.setInput(input);
@@ -98,9 +101,10 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
@SuppressWarnings("unchecked")
Keys.SelectorFunctionKeys<T, ?> selectorKeys = (Keys.SelectorFunctionKeys<T, ?>) keys;
-
+
+
PlanUnwrappingReduceGroupOperator<T, T, ?> po = translateSelectorFunctionDistinct(
- selectorKeys, function, getInputType(), getResultType(), name, input, true);
+ selectorKeys, function, combineFunction, getInputType(), getResultType(), name, input, true);
po.setDegreeOfParallelism(this.getParallelism());
@@ -114,7 +118,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
// --------------------------------------------------------------------------------------------
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionDistinct(
- Keys.SelectorFunctionKeys<IN, ?> rawKeys, GroupReduceFunction<IN, OUT> function,
+ Keys.SelectorFunctionKeys<IN, ?> rawKeys, RichGroupReduceFunction<IN, OUT> function, FlatCombineFunction<IN> combineFunction,
TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input,
boolean combinable)
{
@@ -124,10 +128,12 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
TypeInformation<Tuple2<K, IN>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, IN>>(keys.getKeyType(), inputType);
KeyExtractingMapper<IN, K> extractor = new KeyExtractingMapper<IN, K>(keys.getKeyExtractor());
+
+
+ PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
+ new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable);
- PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer = new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable);
-
- MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, GenericMap<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor");
+ MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor");
reducer.setInput(mapper);
mapper.setInput(input);
@@ -138,7 +144,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
return reducer;
}
- public static final class DistinctFunction<T> extends GroupReduceFunction<T, T> {
+ public static final class DistinctFunction<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
@@ -148,4 +154,16 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
out.collect(values.next());
}
}
+
+ public static final class DistinctCombiner<T> implements FlatCombineFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void combine(Iterator<T> values, Collector<T> out)
+ throws Exception {
+ out.collect(values.next());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/FilterOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/FilterOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/FilterOperator.java
index ed4a786..1c03ccd 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/FilterOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/FilterOperator.java
@@ -18,9 +18,9 @@
package org.apache.flink.api.java.operators;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.java.functions.FilterFunction;
import org.apache.flink.api.java.operators.translation.PlanFilterOperator;
import org.apache.flink.api.java.DataSet;
@@ -44,7 +44,7 @@ public class FilterOperator<T> extends SingleInputUdfOperator<T, T, FilterOperat
}
@Override
- protected org.apache.flink.api.common.operators.base.FilterOperatorBase<T, GenericFlatMap<T,T>> translateToDataFlow(Operator<T> input) {
+ protected org.apache.flink.api.common.operators.base.FilterOperatorBase<T, FlatMapFunction<T,T>> translateToDataFlow(Operator<T> input) {
String name = getName() != null ? getName() : function.getClass().getName();
// create operator
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/FlatMapOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/FlatMapOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/FlatMapOperator.java
index 14c0819..8e531d4 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/FlatMapOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/FlatMapOperator.java
@@ -18,11 +18,10 @@
package org.apache.flink.api.java.operators;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
-import org.apache.flink.api.java.functions.FlatMapFunction;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.DataSet;
@@ -47,11 +46,11 @@ public class FlatMapOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, Fl
}
@Override
- protected org.apache.flink.api.common.operators.base.FlatMapOperatorBase<IN, OUT, GenericFlatMap<IN,OUT>> translateToDataFlow(Operator<IN> input) {
+ protected org.apache.flink.api.common.operators.base.FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN,OUT>> translateToDataFlow(Operator<IN> input) {
String name = getName() != null ? getName() : function.getClass().getName();
// create operator
- FlatMapOperatorBase<IN, OUT, GenericFlatMap<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, GenericFlatMap<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
+ FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>> po = new FlatMapOperatorBase<IN, OUT, FlatMapFunction<IN, OUT>>(function, new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType()), name);
// set input
po.setInput(input);
// set dop
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
new file mode 100644
index 0000000..7ab0b11
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.operators;
+
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.UnaryOperatorInformation;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.common.operators.base.MapOperatorBase;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
+import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.types.TypeInformation;
+
+import org.apache.flink.api.java.DataSet;
+
+
+/**
+ * This operator represents the application of a "reduceGroup" function on a data set, and the
+ * result data set produced by the function.
+ *
+ * @param <IN> The type of the data set consumed by the operator.
+ * @param <OUT> The type of the data set created by the operator.
+ */
+public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT, GroupReduceOperator<IN, OUT>> {
+
+ private final GroupReduceFunction<IN, OUT> function;
+
+ private final Grouping<IN> grouper;
+
+ // reduceFunction is a GroupReduceFunction
+ private boolean richFunction;
+
+ private boolean combinable;
+
+ /**
+ * Constructor for a non-grouped reduce (all reduce).
+ *
+ * @param input The input data set to the groupReduce function.
+ * @param function The user-defined GroupReduce function.
+ */
+ public GroupReduceOperator(DataSet<IN> input, GroupReduceFunction<IN, OUT> function) {
+ super(input, TypeExtractor.getGroupReduceReturnTypes(function, input.getType()));
+
+ this.function = function;
+ this.grouper = null;
+
+ checkCombinability();
+ }
+
+ /**
+ * Constructor for a grouped reduce.
+ *
+ * @param input The grouped input to be processed group-wise by the groupReduce function.
+ * @param function The user-defined GroupReduce function.
+ */
+ public GroupReduceOperator(Grouping<IN> input, GroupReduceFunction<IN, OUT> function) {
+ super(input != null ? input.getDataSet() : null, TypeExtractor.getGroupReduceReturnTypes(function, input.getDataSet().getType()));
+
+ this.function = function;
+ this.grouper = input;
+
+ checkCombinability();
+
+ extractSemanticAnnotationsFromUdf(function.getClass());
+ }
+
+ private void checkCombinability() {
+ if (function instanceof FlatCombineFunction &&
+ function.getClass().getAnnotation(RichGroupReduceFunction.Combinable.class) != null) {
+ this.combinable = true;
+ }
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+ // Properties
+ // --------------------------------------------------------------------------------------------
+
+ public boolean isCombinable() {
+ return combinable;
+ }
+
+ public void setCombinable(boolean combinable) {
+ // sanity check that the function is a subclass of the combine interface
+ if (combinable && !(function instanceof FlatCombineFunction)) {
+ throw new IllegalArgumentException("The function does not implement the combine interface.");
+ }
+
+ this.combinable = combinable;
+ }
+
+ @Override
+ protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<?, OUT, ?> translateToDataFlow(Operator<IN> input) {
+
+ String name = getName() != null ? getName() : function.getClass().getName();
+
+ // distinguish between grouped reduce and non-grouped reduce
+ if (grouper == null) {
+ // non grouped reduce
+ UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
+ GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
+ new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, new int[0], name);
+
+ po.setCombinable(combinable);
+ // set input
+ po.setInput(input);
+ // the degree of parallelism for a non grouped reduce can only be 1
+ po.setDegreeOfParallelism(1);
+ return po;
+ }
+
+ if (grouper.getKeys() instanceof Keys.SelectorFunctionKeys) {
+
+ @SuppressWarnings("unchecked")
+ Keys.SelectorFunctionKeys<IN, ?> selectorKeys = (Keys.SelectorFunctionKeys<IN, ?>) grouper.getKeys();
+
+ PlanUnwrappingReduceGroupOperator<IN, OUT, ?> po = translateSelectorFunctionReducer(
+ selectorKeys, function, getInputType(), getResultType(), name, input, isCombinable());
+
+ po.setDegreeOfParallelism(this.getParallelism());
+
+ return po;
+ }
+ else if (grouper.getKeys() instanceof Keys.FieldPositionKeys) {
+
+ int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
+ UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
+ GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
+ new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
+
+ po.setCombinable(combinable);
+ po.setInput(input);
+ po.setDegreeOfParallelism(this.getParallelism());
+
+ // set group order
+ if (grouper instanceof SortedGrouping) {
+ SortedGrouping<IN> sortedGrouper = (SortedGrouping<IN>) grouper;
+
+ int[] sortKeyPositions = sortedGrouper.getGroupSortKeyPositions();
+ Order[] sortOrders = sortedGrouper.getGroupSortOrders();
+
+ Ordering o = new Ordering();
+ for(int i=0; i < sortKeyPositions.length; i++) {
+ o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]);
+ }
+ po.setGroupOrder(o);
+ }
+
+ return po;
+ }
+ else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
+
+ int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
+ UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
+ GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
+ new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
+
+ po.setCombinable(combinable);
+ po.setInput(input);
+ po.setDegreeOfParallelism(this.getParallelism());
+
+ return po;
+ }
+ else {
+ throw new UnsupportedOperationException("Unrecognized key type.");
+ }
+
+ }
+
+
+ // --------------------------------------------------------------------------------------------
+
+ private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
+ Keys.SelectorFunctionKeys<IN, ?> rawKeys, GroupReduceFunction<IN, OUT> function,
+ TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input,
+ boolean combinable)
+ {
+ @SuppressWarnings("unchecked")
+ final Keys.SelectorFunctionKeys<IN, K> keys = (Keys.SelectorFunctionKeys<IN, K>) rawKeys;
+
+ TypeInformation<Tuple2<K, IN>> typeInfoWithKey = new TupleTypeInfo<Tuple2<K, IN>>(keys.getKeyType(), inputType);
+
+ KeyExtractingMapper<IN, K> extractor = new KeyExtractingMapper<IN, K>(keys.getKeyExtractor());
+
+ PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer = new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable);
+
+ MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor");
+
+ reducer.setInput(mapper);
+ mapper.setInput(input);
+
+ // set the mapper's parallelism to the input parallelism to make sure it is chained
+ mapper.setDegreeOfParallelism(input.getDegreeOfParallelism());
+
+ return reducer;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
index 37e74ef..3223f4d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
@@ -26,8 +26,8 @@ import org.apache.flink.api.java.DataSet;
* Grouping is an intermediate step for a transformation on a grouped DataSet.<br/>
* The following transformation can be applied on Grouping:
* <ul>
- * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.ReduceFunction)},</li>
- * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.GroupReduceFunction)}, and</li>
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)},</li>
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)}, and</li>
* <li>{@link UnsortedGrouping#aggregate(org.apache.flink.api.java.aggregation.Aggregations, int)}.</li>
* </ul>
*
[35/39] git commit: [FLINK-909] Adjust aggregators test case
Posted by rm...@apache.org.
[FLINK-909] Adjust aggregators test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/b6ae1ef3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/b6ae1ef3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/b6ae1ef3
Branch: refs/heads/travis_test
Commit: b6ae1ef3d2f95db69c91203b138baf43bd030d93
Parents: 76a48df
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Aug 8 14:47:43 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Aug 8 14:47:43 2014 +0200
----------------------------------------------------------------------
.../runtime/iterative/concurrent/SuperstepKickoffLatch.java | 2 +-
.../iterative/concurrent/SuperstepKickoffLatchBroker.java | 1 -
.../ConnectedComponentsWithParametrizableAggregatorITCase.java | 5 ++++-
3 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b6ae1ef3/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
index b53928c..91b1db0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatch.java
@@ -29,7 +29,7 @@ public class SuperstepKickoffLatch {
public void triggerNextSuperstep() {
synchronized (monitor) {
if (terminated) {
- throw new IllegalStateException("Already teriminated.");
+ throw new IllegalStateException("Already terminated.");
}
superstepNumber++;
monitor.notifyAll();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b6ae1ef3/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
index 41f6985..4439acf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/concurrent/SuperstepKickoffLatchBroker.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.iterative.concurrent;
public class SuperstepKickoffLatchBroker extends Broker<SuperstepKickoffLatch> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b6ae1ef3/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
index 9382708..e1ae706 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
@@ -96,11 +96,14 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
protected void postSubmit() throws Exception {
compareResultsByLinesInMemory(expectedResult, resultPath);
long[] aggr_values = ConnectedComponentsWithAggregatorProgram.aggr_value;
+
+ // note that position 0 has the end result from superstep 1, retrieved at the start of iteration 2
+ // position one as superstep 2, retrieved at the start of iteration 3.
+ // the result from iteration 5 is not available, because no iteration 6 happens
Assert.assertEquals(3, aggr_values[0]);
Assert.assertEquals(4, aggr_values[1]);
Assert.assertEquals(5, aggr_values[2]);
Assert.assertEquals(6, aggr_values[3]);
- Assert.assertEquals(6, aggr_values[4]);
}
[26/39] git commit: [FLINK-701] Change KeySelector to a SAM interface
Posted by rm...@apache.org.
[FLINK-701] Change KeySelector to a SAM interface
This closes #85
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/934e4e00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/934e4e00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/934e4e00
Branch: refs/heads/travis_test
Commit: 934e4e00df012b4aab128294c05153d0c46f9887
Parents: bc89e91
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 31 20:31:54 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Aug 1 02:33:34 2014 +0200
----------------------------------------------------------------------
.../testfunctions/IdentityKeyExtractor.java | 3 +-
.../flink/api/java/functions/KeySelector.java | 10 ++--
.../flink/api/java/typeutils/TypeExtractor.java | 9 +++-
.../java/type/extractor/TypeExtractorTest.java | 23 +++++++++
.../lambdas/KeySelectorTest.java | 52 ++++++++++++++++++++
5 files changed, 90 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/934e4e00/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityKeyExtractor.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityKeyExtractor.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityKeyExtractor.java
index 39ef821..7e7ba16 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityKeyExtractor.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityKeyExtractor.java
@@ -16,12 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.testfunctions;
import org.apache.flink.api.java.functions.KeySelector;
-public class IdentityKeyExtractor<T> extends KeySelector<T, T> {
+public class IdentityKeyExtractor<T> implements KeySelector<T, T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/934e4e00/flink-java/src/main/java/org/apache/flink/api/java/functions/KeySelector.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/KeySelector.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/KeySelector.java
index ede7c32..1dcc8c6 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/KeySelector.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/KeySelector.java
@@ -18,6 +18,7 @@
package org.apache.flink.api.java.functions;
+import org.apache.flink.api.common.functions.Function;
/**
* The {@link KeySelector} allows to use arbitrary objects for operations such as
@@ -28,9 +29,7 @@ package org.apache.flink.api.java.functions;
* @param <IN> Type of objects to extract the key from.
* @param <KEY> Type of key.
*/
-public abstract class KeySelector<IN, KEY> implements java.io.Serializable {
-
- private static final long serialVersionUID = 1L;
+public interface KeySelector<IN, KEY> extends Function, java.io.Serializable {
/**
* User-defined function that extracts the key from an arbitrary object.
@@ -54,6 +53,9 @@ public abstract class KeySelector<IN, KEY> implements java.io.Serializable {
*
* @param value The object to get the key from.
* @return The extracted key.
+ *
+ * @throws Exception Throwing an exception will cause the execution of the respective task to fail,
+ * and trigger recovery or cancellation of the program.
*/
- public abstract KEY getKey(IN value);
+ KEY getKey(IN value) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/934e4e00/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index e8ee0bb..d03cc49 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -38,9 +38,11 @@ import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.java.functions.InvalidTypesException;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.types.TypeInformation;
import org.apache.flink.types.Value;
@@ -134,6 +136,10 @@ public class TypeExtractor {
@SuppressWarnings("unchecked")
public static <IN, OUT> TypeInformation<OUT> getKeySelectorTypes(KeySelector<IN, OUT> selectorInterface, TypeInformation<IN> inType) {
+ if (FunctionUtils.isLambdaFunction(selectorInterface)) {
+ throw new UnsupportedLambdaExpressionException();
+ }
+
validateInputType(KeySelector.class, selectorInterface.getClass(), 0, inType);
if(selectorInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) selectorInterface).getProducedType();
@@ -406,7 +412,8 @@ public class TypeExtractor {
return parameter;
}
- throw new IllegalArgumentException(baseClass.getName() + " must be implemented.");
+ throw new IllegalArgumentException("The types of the interface " + baseClass.getName() + " could not be inferred. " +
+ "Support for synthetic interfaces, lambdas, and generic types is limited at this point.");
}
private static Type getParameterTypeFromGenericType(Class<?> baseClass, ArrayList<Type> typeHierarchy, Type t, int pos) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/934e4e00/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index 8346d00..d5044a8 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -1430,4 +1430,27 @@ public class TypeExtractorTest {
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(mapInterface, BasicTypeInfo.STRING_TYPE_INFO);
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti);
}
+
+ @SuppressWarnings({ "serial", "unchecked", "rawtypes" })
+ @Test
+ public void testExtractKeySelector() {
+ KeySelector<String, Integer> selector = new KeySelector<String, Integer>() {
+ @Override
+ public Integer getKey(String value) { return null; }
+ };
+
+ TypeInformation<?> ti = TypeExtractor.getKeySelectorTypes(selector, BasicTypeInfo.STRING_TYPE_INFO);
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ti);
+
+ try {
+ TypeExtractor.getKeySelectorTypes((KeySelector) selector, BasicTypeInfo.BOOLEAN_TYPE_INFO);
+ Assert.fail();
+ }
+ catch (InvalidTypesException e) {
+ // good
+ }
+ catch (Exception e) {
+ Assert.fail("wrong exception type");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/934e4e00/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/KeySelectorTest.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/KeySelectorTest.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/KeySelectorTest.java
new file mode 100644
index 0000000..fd61e25
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/KeySelectorTest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.BasicTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.junit.Assert;
+
+public class KeySelectorTest {
+
+ public void testSelectorLambda() {
+ try {
+ KeySelector<Tuple2<String, Integer>, String> selector = (t) -> t.f0;
+
+ try {
+ TypeExtractor.getKeySelectorTypes(selector,
+ new TupleTypeInfo<Tuple2<String, Integer>>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO));
+ Assert.fail("No unsupported lambdas exception");
+ }
+ catch (UnsupportedLambdaExpressionException e) {
+ // good
+ }
+ catch (Exception e) {
+ Assert.fail("Wrong exception type");
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+}
[02/39] git commit: Add ASM to LICENSE file
Posted by rm...@apache.org.
Add ASM to LICENSE file
[ci skip]
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/c0661900
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/c0661900
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/c0661900
Branch: refs/heads/travis_test
Commit: c0661900e0948445d54ed048f6671b85759f62d7
Parents: dc55cc3
Author: uce <u....@fu-berlin.de>
Authored: Wed Jul 23 09:05:44 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Wed Jul 23 09:05:44 2014 +0200
----------------------------------------------------------------------
LICENSE | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c0661900/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index aa537d7..ed91114 100644
--- a/LICENSE
+++ b/LICENSE
@@ -299,7 +299,7 @@ under BSD-style licenses
- Scala Library (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
- Scala Compiler (BSD-like) - (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
- Scala Compiler Reflect (BSD-like) - (http://www.scala-lang.org/) - Copyright (c) 2002-2014 EPFL, Copyright (c) 2011-2014 Typesafe, Inc.
-
+ - ASM (BSD-like) - (http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
(Below is the 3-clause BSD license)
[29/39] [FLINK-1023] Switch group-at-a-time function to
java.lang.Iterable (from java.util.Iterator) Iterables over transient data
throw an TraversableOnceException when iterated over again.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/record/operators/ReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/ReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/ReduceOperator.java
index 3860fbe..074a70d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/ReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/ReduceOperator.java
@@ -16,9 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.api.java.record.operators;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -29,6 +31,8 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.RecordOperator;
@@ -36,17 +40,20 @@ import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.java.operators.translation.WrappingFunction;
import org.apache.flink.api.java.record.functions.FunctionAnnotation;
import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.types.Key;
import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
/**
* ReduceOperator evaluating a {@link ReduceFunction} over each group of records that share the same key.
*
* @see ReduceFunction
*/
-public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, ReduceFunction> implements RecordOperator {
+public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, GroupReduceFunction<Record, Record>> implements RecordOperator {
private static final String DEFAULT_NAME = "<Unnamed Reducer>"; // the default name for contracts
@@ -55,6 +62,8 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
*/
private final Class<? extends Key<?>>[] keyTypes;
+ private final UserCodeWrapper<ReduceFunction> originalFunction;
+
// --------------------------------------------------------------------------------------------
/**
@@ -63,7 +72,11 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
* @param udf The {@link ReduceFunction} implementation for this Reduce contract.
*/
public static Builder builder(ReduceFunction udf) {
- return new Builder(new UserCodeObjectWrapper<ReduceFunction>(udf));
+ UserCodeWrapper<ReduceFunction> original = new UserCodeObjectWrapper<ReduceFunction>(udf);
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> wrapped =
+ new UserCodeObjectWrapper<GroupReduceFunction<Record, Record>>(new WrappingReduceFunction(udf));
+
+ return new Builder(original, wrapped);
}
/**
@@ -74,7 +87,11 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
* @param keyColumn The position of the key.
*/
public static Builder builder(ReduceFunction udf, Class<? extends Key<?>> keyClass, int keyColumn) {
- return new Builder(new UserCodeObjectWrapper<ReduceFunction>(udf), keyClass, keyColumn);
+ UserCodeWrapper<ReduceFunction> original = new UserCodeObjectWrapper<ReduceFunction>(udf);
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> wrapped =
+ new UserCodeObjectWrapper<GroupReduceFunction<Record, Record>>(new WrappingReduceFunction(udf));
+
+ return new Builder(original, wrapped, keyClass, keyColumn);
}
/**
@@ -83,7 +100,11 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
* @param udf The {@link ReduceFunction} implementation for this Reduce contract.
*/
public static Builder builder(Class<? extends ReduceFunction> udf) {
- return new Builder(new UserCodeClassWrapper<ReduceFunction>(udf));
+ UserCodeWrapper<ReduceFunction> original = new UserCodeClassWrapper<ReduceFunction>(udf);
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> wrapped =
+ new UserCodeObjectWrapper<GroupReduceFunction<Record, Record>>(new WrappingClassReduceFunction(udf));
+
+ return new Builder(original, wrapped);
}
/**
@@ -94,7 +115,11 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
* @param keyColumn The position of the key.
*/
public static Builder builder(Class<? extends ReduceFunction> udf, Class<? extends Key<?>> keyClass, int keyColumn) {
- return new Builder(new UserCodeClassWrapper<ReduceFunction>(udf), keyClass, keyColumn);
+ UserCodeWrapper<ReduceFunction> original = new UserCodeClassWrapper<ReduceFunction>(udf);
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> wrapped =
+ new UserCodeObjectWrapper<GroupReduceFunction<Record, Record>>(new WrappingClassReduceFunction(udf));
+
+ return new Builder(original, wrapped, keyClass, keyColumn);
}
/**
@@ -102,8 +127,10 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
* @param builder
*/
protected ReduceOperator(Builder builder) {
- super(builder.udf, OperatorInfoHelper.unary(), builder.getKeyColumnsArray(), builder.name);
+ super(builder.udfWrapper, OperatorInfoHelper.unary(), builder.getKeyColumnsArray(), builder.name);
+
this.keyTypes = builder.getKeyClassesArray();
+ this.originalFunction = builder.originalUdf;
if (builder.inputs != null && !builder.inputs.isEmpty()) {
setInput(Operator.createUnionCascade(builder.inputs));
@@ -111,7 +138,8 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
setGroupOrder(builder.secondaryOrder);
setBroadcastVariables(builder.broadcastInputs);
- setSemanticProperties(FunctionAnnotation.readSingleConstantAnnotations(builder.udf));
+
+ setSemanticProperties(FunctionAnnotation.readSingleConstantAnnotations(originalFunction));
}
// --------------------------------------------------------------------------------------------
@@ -126,7 +154,7 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
@Override
public boolean isCombinable() {
- return super.isCombinable() || getUserCodeWrapper().getUserCodeAnnotation(Combinable.class) != null;
+ return super.isCombinable() || originalFunction.getUserCodeAnnotation(Combinable.class) != null;
}
/**
@@ -178,7 +206,8 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
public static class Builder {
/* The required parameters */
- private final UserCodeWrapper<ReduceFunction> udf;
+ private final UserCodeWrapper<ReduceFunction> originalUdf;
+ private final UserCodeWrapper<GroupReduceFunction<Record, Record>> udfWrapper;
private final List<Class<? extends Key<?>>> keyClasses;
private final List<Integer> keyColumns;
@@ -191,10 +220,11 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
/**
* Creates a Builder with the provided {@link ReduceFunction} implementation.
*
- * @param udf The {@link ReduceFunction} implementation for this Reduce contract.
+ * @param wrappedUdf The {@link ReduceFunction} implementation for this Reduce contract.
*/
- private Builder(UserCodeWrapper<ReduceFunction> udf) {
- this.udf = udf;
+ private Builder(UserCodeWrapper<ReduceFunction> originalUdf, UserCodeWrapper<GroupReduceFunction<Record, Record>> wrappedUdf) {
+ this.originalUdf = originalUdf;
+ this.udfWrapper = wrappedUdf;
this.keyClasses = new ArrayList<Class<? extends Key<?>>>();
this.keyColumns = new ArrayList<Integer>();
this.inputs = new ArrayList<Operator<Record>>();
@@ -204,12 +234,16 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
/**
* Creates a Builder with the provided {@link ReduceFunction} implementation.
*
- * @param udf The {@link ReduceFunction} implementation for this Reduce contract.
+ * @param wrappedUdf The {@link ReduceFunction} implementation for this Reduce contract.
* @param keyClass The class of the key data type.
* @param keyColumn The position of the key.
*/
- private Builder(UserCodeWrapper<ReduceFunction> udf, Class<? extends Key<?>> keyClass, int keyColumn) {
- this.udf = udf;
+ private Builder(UserCodeWrapper<ReduceFunction> originalUdf,
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> wrappedUdf,
+ Class<? extends Key<?>> keyClass, int keyColumn)
+ {
+ this.originalUdf = originalUdf;
+ this.udfWrapper = wrappedUdf;
this.keyClasses = new ArrayList<Class<? extends Key<?>>>();
this.keyClasses.add(keyClass);
this.keyColumns = new ArrayList<Integer>();
@@ -325,9 +359,48 @@ public class ReduceOperator extends GroupReduceOperatorBase<Record, Record, Redu
*/
public ReduceOperator build() {
if (name == null) {
- name = udf.getUserCodeClass().getName();
+ name = udfWrapper.getUserCodeClass().getName();
}
return new ReduceOperator(this);
}
}
+
+ // ============================================================================================
+
+ public static class WrappingReduceFunction extends WrappingFunction<ReduceFunction> implements GroupReduceFunction<Record, Record>, FlatCombineFunction<Record> {
+
+ private static final long serialVersionUID = 1L;
+
+ public WrappingReduceFunction(ReduceFunction reducer) {
+ super(reducer);
+ }
+
+ @Override
+ public final void reduce(Iterable<Record> records, Collector<Record> out) throws Exception {
+ this.wrappedFunction.reduce(records.iterator(), out);
+ }
+
+ @Override
+ public final void combine(Iterable<Record> records, Collector<Record> out) throws Exception {
+ this.wrappedFunction.combine(records.iterator(), out);
+ }
+ }
+
+ public static final class WrappingClassReduceFunction extends WrappingReduceFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ public WrappingClassReduceFunction(Class<? extends ReduceFunction> reducer) {
+ super(InstantiationUtil.instantiate(reducer));
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeObject(wrappedFunction.getClass());
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ Class<?> clazz = (Class<?>) in.readObject();
+ this.wrappedFunction = (ReduceFunction) InstantiationUtil.instantiate(clazz);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
index be872e5..9fcf963 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
@@ -246,7 +246,7 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
public static class SolutionWorksetCoGroup1 extends RichCoGroupFunction<Tuple2<Double, String>, Tuple3<Double, Long, String>, Tuple3<Double, Long, String>> {
@Override
- public void coGroup(Iterator<Tuple2<Double, String>> first, Iterator<Tuple3<Double, Long, String>> second,
+ public void coGroup(Iterable<Tuple2<Double, String>> first, Iterable<Tuple3<Double, Long, String>> second,
Collector<Tuple3<Double, Long, String>> out) {
}
}
@@ -254,7 +254,7 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
public static class SolutionWorksetCoGroup2 extends RichCoGroupFunction<Tuple3<Double, Long, String>, Tuple2<Double, String>, Tuple3<Double, Long, String>> {
@Override
- public void coGroup(Iterator<Tuple3<Double, Long, String>> second, Iterator<Tuple2<Double, String>> first,
+ public void coGroup(Iterable<Tuple3<Double, Long, String>> second, Iterable<Tuple2<Double, String>> first,
Collector<Tuple3<Double, Long, String>> out) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/test/java/org/apache/flink/api/java/record/CoGroupWrappingFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/CoGroupWrappingFunctionTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/CoGroupWrappingFunctionTest.java
new file mode 100644
index 0000000..78a7ad8
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/CoGroupWrappingFunctionTest.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.operators.DualInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.java.record.functions.CoGroupFunction;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsFirst;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFieldsSecond;
+import org.apache.flink.api.java.record.operators.CoGroupOperator;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class CoGroupWrappingFunctionTest {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWrappedCoGroupObject() {
+ try {
+ AtomicInteger methodCounter = new AtomicInteger();
+
+ CoGroupOperator coGroupOp = CoGroupOperator.builder(new TestCoGroupFunction(methodCounter), LongValue.class, 1, 2).build();
+
+ RichFunction cogrouper = (RichFunction) coGroupOp.getUserCodeWrapper().getUserCodeObject();
+
+ // test the method invocations
+ cogrouper.close();
+ cogrouper.open(new Configuration());
+ assertEquals(2, methodCounter.get());
+
+ // prepare the coGroup
+ final List<Record> target = new ArrayList<Record>();
+ Collector<Record> collector = new Collector<Record>() {
+ @Override
+ public void collect(Record record) {
+ target.add(record);
+ }
+ @Override
+ public void close() {}
+ };
+
+ List<Record> source1 = new ArrayList<Record>();
+ source1.add(new Record(new IntValue(42)));
+ source1.add(new Record(new IntValue(13)));
+
+ List<Record> source2 = new ArrayList<Record>();
+ source2.add(new Record(new LongValue(11)));
+ source2.add(new Record(new LongValue(17)));
+
+ // test coGroup
+ ((org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>) cogrouper).coGroup(source1, source2, collector);
+ assertEquals(4, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(2).getField(0, LongValue.class));
+ assertEquals(new LongValue(17), target.get(3).getField(0, LongValue.class));
+ target.clear();
+
+ // test the serialization
+ SerializationUtils.clone((java.io.Serializable) cogrouper);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testWrappedCoGroupClass() {
+ try {
+ CoGroupOperator coGroupOp = CoGroupOperator.builder(TestCoGroupFunction.class, LongValue.class, 1, 2).build();
+
+ UserCodeWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> udf = coGroupOp.getUserCodeWrapper();
+ UserCodeWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> copy = SerializationUtils.clone(udf);
+ org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record> cogrouper = copy.getUserCodeObject();
+
+ // prepare the coGpuü
+ final List<Record> target = new ArrayList<Record>();
+ Collector<Record> collector = new Collector<Record>() {
+ @Override
+ public void collect(Record record) {
+ target.add(record);
+ }
+ @Override
+ public void close() {}
+ };
+
+ List<Record> source1 = new ArrayList<Record>();
+ source1.add(new Record(new IntValue(42)));
+ source1.add(new Record(new IntValue(13)));
+
+ List<Record> source2 = new ArrayList<Record>();
+ source2.add(new Record(new LongValue(11)));
+ source2.add(new Record(new LongValue(17)));
+
+ // test coGroup
+ cogrouper.coGroup(source1, source2, collector);
+ assertEquals(4, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(2).getField(0, LongValue.class));
+ assertEquals(new LongValue(17), target.get(3).getField(0, LongValue.class));
+ target.clear();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExtractSemantics() {
+ try {
+ {
+ CoGroupOperator coGroupOp = CoGroupOperator.builder(new TestCoGroupFunction(), LongValue.class, 1, 2).build();
+
+ DualInputSemanticProperties props = coGroupOp.getSemanticProperties();
+ FieldSet fw2 = props.getForwardedField1(2);
+ FieldSet fw4 = props.getForwardedField2(4);
+
+ assertNotNull(fw2);
+ assertNotNull(fw4);
+ assertEquals(1, fw2.size());
+ assertEquals(1, fw4.size());
+ assertTrue(fw2.contains(2));
+ assertTrue(fw4.contains(4));
+ }
+ {
+ CoGroupOperator coGroupOp = CoGroupOperator.builder(TestCoGroupFunction.class, LongValue.class, 1, 2).build();
+
+ DualInputSemanticProperties props = coGroupOp.getSemanticProperties();
+ FieldSet fw2 = props.getForwardedField1(2);
+ FieldSet fw4 = props.getForwardedField2(4);
+
+ assertNotNull(fw2);
+ assertNotNull(fw4);
+ assertEquals(1, fw2.size());
+ assertEquals(1, fw4.size());
+ assertTrue(fw2.contains(2));
+ assertTrue(fw4.contains(4));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @ConstantFieldsFirst(2)
+ @ConstantFieldsSecond(4)
+ public static class TestCoGroupFunction extends CoGroupFunction {
+
+ private final AtomicInteger methodCounter;
+
+ private TestCoGroupFunction(AtomicInteger methodCounter) {
+ this.methodCounter= methodCounter;
+ }
+
+ public TestCoGroupFunction() {
+ methodCounter = new AtomicInteger();
+ }
+
+ @Override
+ public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) throws Exception {
+ while (records1.hasNext()) {
+ out.collect(records1.next());
+ }
+ while (records2.hasNext()) {
+ out.collect(records2.next());
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ methodCounter.incrementAndGet();
+ super.close();
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ methodCounter.incrementAndGet();
+ super.open(parameters);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/test/java/org/apache/flink/api/java/record/ReduceWrappingFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/record/ReduceWrappingFunctionTest.java b/flink-java/src/test/java/org/apache/flink/api/java/record/ReduceWrappingFunctionTest.java
new file mode 100644
index 0000000..653bb16
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/record/ReduceWrappingFunctionTest.java
@@ -0,0 +1,246 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.record;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
+import org.apache.flink.api.common.operators.util.FieldSet;
+import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFields;
+import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.record.operators.ReduceOperator;
+import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.types.IntValue;
+import org.apache.flink.types.LongValue;
+import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class ReduceWrappingFunctionTest {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWrappedReduceObject() {
+ try {
+ AtomicInteger methodCounter = new AtomicInteger();
+
+ ReduceOperator reduceOp = ReduceOperator.builder(new TestReduceFunction(methodCounter)).build();
+
+ RichFunction reducer = (RichFunction) reduceOp.getUserCodeWrapper().getUserCodeObject();
+
+ // test the method invocations
+ reducer.close();
+ reducer.open(new Configuration());
+ assertEquals(2, methodCounter.get());
+
+ // prepare the reduce / combine tests
+ final List<Record> target = new ArrayList<Record>();
+ Collector<Record> collector = new Collector<Record>() {
+ @Override
+ public void collect(Record record) {
+ target.add(record);
+ }
+ @Override
+ public void close() {}
+ };
+
+ List<Record> source = new ArrayList<Record>();
+ source.add(new Record(new IntValue(42), new LongValue(11)));
+ source.add(new Record(new IntValue(13), new LongValue(17)));
+
+ // test reduce
+ ((GroupReduceFunction<Record, Record>) reducer).reduce(source, collector);
+ assertEquals(2, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(0).getField(1, LongValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(17), target.get(1).getField(1, LongValue.class));
+ target.clear();
+
+ // test combine
+ ((FlatCombineFunction<Record>) reducer).combine(source, collector);
+ assertEquals(2, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(0).getField(1, LongValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(17), target.get(1).getField(1, LongValue.class));
+ target.clear();
+
+ // test the serialization
+ SerializationUtils.clone((java.io.Serializable) reducer);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testWrappedReduceClass() {
+ try {
+ ReduceOperator reduceOp = ReduceOperator.builder(TestReduceFunction.class).build();
+
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> udf = reduceOp.getUserCodeWrapper();
+ UserCodeWrapper<GroupReduceFunction<Record, Record>> copy = SerializationUtils.clone(udf);
+ GroupReduceFunction<Record, Record> reducer = copy.getUserCodeObject();
+
+ // prepare the reduce / combine tests
+ final List<Record> target = new ArrayList<Record>();
+ Collector<Record> collector = new Collector<Record>() {
+ @Override
+ public void collect(Record record) {
+ target.add(record);
+ }
+ @Override
+ public void close() {}
+ };
+
+ List<Record> source = new ArrayList<Record>();
+ source.add(new Record(new IntValue(42), new LongValue(11)));
+ source.add(new Record(new IntValue(13), new LongValue(17)));
+
+ // test reduce
+ reducer.reduce(source, collector);
+ assertEquals(2, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(0).getField(1, LongValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(17), target.get(1).getField(1, LongValue.class));
+ target.clear();
+
+ // test combine
+ ((FlatCombineFunction<Record>) reducer).combine(source, collector);
+ assertEquals(2, target.size());
+ assertEquals(new IntValue(42), target.get(0).getField(0, IntValue.class));
+ assertEquals(new LongValue(11), target.get(0).getField(1, LongValue.class));
+ assertEquals(new IntValue(13), target.get(1).getField(0, IntValue.class));
+ assertEquals(new LongValue(17), target.get(1).getField(1, LongValue.class));
+ target.clear();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExtractSemantics() {
+ try {
+ {
+ ReduceOperator reduceOp = ReduceOperator.builder(new TestReduceFunction()).build();
+
+ SingleInputSemanticProperties props = reduceOp.getSemanticProperties();
+ FieldSet fw2 = props.getForwardedField(2);
+ FieldSet fw4 = props.getForwardedField(4);
+
+ assertNotNull(fw2);
+ assertNotNull(fw4);
+ assertEquals(1, fw2.size());
+ assertEquals(1, fw4.size());
+ assertTrue(fw2.contains(2));
+ assertTrue(fw4.contains(4));
+ }
+ {
+ ReduceOperator reduceOp = ReduceOperator.builder(TestReduceFunction.class).build();
+
+ SingleInputSemanticProperties props = reduceOp.getSemanticProperties();
+ FieldSet fw2 = props.getForwardedField(2);
+ FieldSet fw4 = props.getForwardedField(4);
+
+ assertNotNull(fw2);
+ assertNotNull(fw4);
+ assertEquals(1, fw2.size());
+ assertEquals(1, fw4.size());
+ assertTrue(fw2.contains(2));
+ assertTrue(fw4.contains(4));
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCombinable() {
+ try {
+ {
+ ReduceOperator reduceOp = ReduceOperator.builder(new TestReduceFunction()).build();
+ assertTrue(reduceOp.isCombinable());
+ }
+ {
+ ReduceOperator reduceOp = ReduceOperator.builder(TestReduceFunction.class).build();
+ assertTrue(reduceOp.isCombinable());
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Combinable
+ @ConstantFields({2, 4})
+ public static class TestReduceFunction extends ReduceFunction {
+
+ private final AtomicInteger methodCounter;
+
+ private TestReduceFunction(AtomicInteger methodCounter) {
+ this.methodCounter= methodCounter;
+ }
+
+ public TestReduceFunction() {
+ methodCounter = new AtomicInteger();
+ }
+
+ @Override
+ public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
+ while (records.hasNext()) {
+ out.collect(records.next());
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ methodCounter.incrementAndGet();
+ super.close();
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ methodCounter.incrementAndGet();
+ super.open(parameters);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index d5044a8..c8a8ee9 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -21,7 +21,6 @@ package org.apache.flink.api.java.type.extractor;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Iterator;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
@@ -72,7 +71,7 @@ public class TypeExtractorTest {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Boolean> values, Collector<Boolean> out) throws Exception {
+ public void reduce(Iterable<Boolean> values, Collector<Boolean> out) throws Exception {
// nothing to do
}
};
@@ -1084,7 +1083,7 @@ public class TypeExtractorTest {
private static final long serialVersionUID = 1L;
@Override
- public void coGroup(Iterator<String[]> first, Iterator<String[]> second, Collector<String[]> out) throws Exception {
+ public void coGroup(Iterable<String[]> first, Iterable<String[]> second, Collector<String[]> out) throws Exception {
// nothing to do
}
};
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
index 7dd4dea..ab18bf8 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
@@ -48,14 +48,11 @@ public class CoGroupITCase implements Serializable {
DataSet<Tuple2<Integer,String>> joined = left.coGroup(right).where(0).equalTo(0)
.with((values1, values2, out) -> {
int sum = 0;
- String conc = "";
- while (values1.hasNext()) {
- sum += values1.next().f0;
- conc += values1.next().f1;
+ for (Tuple2<Integer, String> next : values1) {
+ sum += next.f0;
}
- while (values2.hasNext()) {
- sum += values2.next().f0;
- conc += values2.next().f1;
+ for (Tuple2<Integer, String> next : values2) {
+ sum += next.f0;
}
});
env.execute();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
index a86de1f..7b5d659 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
@@ -38,8 +38,7 @@ public class GroupReduceITCase implements Serializable {
DataSet<String> stringDs = env.fromElements("aa", "ab", "ac", "ad");
DataSet<String> concatDs = stringDs.reduceGroup((values, out) -> {
String conc = "";
- while (values.hasNext()) {
- String s = values.next();
+ for (String s : values) {
conc = conc.concat(s);
}
out.collect(conc);
@@ -68,9 +67,8 @@ public class GroupReduceITCase implements Serializable {
.groupBy(0)
.reduceGroup((values, out) -> {
String conc = "";
- while (values.hasNext()) {
- String s = values.next().f1;
- conc = conc.concat(s);
+ for (Tuple2<Integer,String> next : values) {
+ conc = conc.concat(next.f1);
}
out.collect(conc);
});
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
index 8761a2e..8590b78 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetFirstDriver.java
@@ -16,10 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
-import java.util.Iterator;
+import java.util.Collections;
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
@@ -29,7 +28,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetBroker;
import org.apache.flink.runtime.iterative.task.AbstractIterativePactTask;
import org.apache.flink.runtime.operators.hash.CompactingHashTable;
-import org.apache.flink.runtime.util.EmptyIterator;
import org.apache.flink.runtime.util.KeyGroupedIterator;
import org.apache.flink.runtime.util.SingleElementIterator;
import org.apache.flink.util.Collector;
@@ -132,7 +130,7 @@ public class CoGroupWithSolutionSetFirstDriver<IT1, IT2, OT> implements Resettab
final KeyGroupedIterator<IT2> probeSideInput = new KeyGroupedIterator<IT2>(taskContext.<IT2>getInput(0), probeSideSerializer, probeSideComparator);
final SingleElementIterator<IT1> siIter = new SingleElementIterator<IT1>();
- final Iterator<IT1> emptySolutionSide = EmptyIterator.<IT1>get();
+ final Iterable<IT1> emptySolutionSide = Collections.emptySet();
final CompactingHashTable<IT1>.HashTableProber<IT2> prober = join.getProber(this.probeSideComparator, this.pairComparator);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
index f2020c7..b3c0ece 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/CoGroupWithSolutionSetSecondDriver.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
-import java.util.Iterator;
-
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
@@ -132,7 +129,7 @@ public class CoGroupWithSolutionSetSecondDriver<IT1, IT2, OT> implements Resetta
final KeyGroupedIterator<IT1> probeSideInput = new KeyGroupedIterator<IT1>(taskContext.<IT1>getInput(0), probeSideSerializer, probeSideComparator);
final SingleElementIterator<IT2> siIter = new SingleElementIterator<IT2>();
- final Iterator<IT2> emptySolutionSide = EmptyIterator.<IT2>get();
+ final Iterable<IT2> emptySolutionSide = EmptyIterator.<IT2>get();
final CompactingHashTable<IT2>.HashTableProber<IT1> prober = join.getProber(this.probeSideComparator, this.pairComparator);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index c8f217c..9bc1893 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -697,7 +697,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
S stub = config.<S>getStubWrapper(this.userCodeClassLoader).getUserCodeObject(stubSuperClass, this.userCodeClassLoader);
// check if the class is a subclass, if the check is required
if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
- Thread.dumpStack();
throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" +
stubSuperClass.getName() + "' as is required.");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
index da8a11b..6a6f6ff 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMerger.java
@@ -48,6 +48,7 @@ import org.apache.flink.runtime.util.EmptyMutableObjectIterator;
import org.apache.flink.runtime.util.KeyGroupedIterator;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
+import org.apache.flink.util.TraversableOnceException;
/**
@@ -503,7 +504,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
* This class implements an iterator over values from a sort buffer. The iterator returns the values of a given
* interval.
*/
- private static final class CombineValueIterator<E> implements Iterator<E> {
+ private static final class CombineValueIterator<E> implements Iterator<E>, Iterable<E> {
private final InMemorySorter<E> buffer; // the buffer from which values are returned
@@ -512,6 +513,8 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
private int last; // the position of the last value to be returned
private int position; // the position of the next value to be returned
+
+ private boolean iteratorAvailable;
/**
* Creates an iterator over the values in a <tt>BufferSortable</tt>.
@@ -535,6 +538,7 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
public void set(int first, int last) {
this.last = last;
this.position = first;
+ this.iteratorAvailable = true;
}
@Override
@@ -564,6 +568,16 @@ public class CombiningUnilateralSortMerger<E> extends UnilateralSortMerger<E> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<E> iterator() {
+ if (iteratorAvailable) {
+ iteratorAvailable = false;
+ return this;
+ } else {
+ throw new TraversableOnceException();
+ }
+ }
};
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
index f575076..ee2a2c9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/FixedLengthRecordSorter.java
@@ -410,7 +410,7 @@ public final class FixedLengthRecordSorter<T> implements InMemorySorter<T> {
*
* @param output The output view to write the records to.
* @param start The logical start position of the subset.
- * @param len The number of elements to write.
+ * @param num The number of elements to write.
* @throws IOException Thrown, if an I/O exception occurred writing to the output view.
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/InMemorySorter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/InMemorySorter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/InMemorySorter.java
index a018def..6f75490 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/InMemorySorter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/InMemorySorter.java
@@ -103,7 +103,7 @@ public interface InMemorySorter<T> extends IndexedSortable {
*
* @param output The output view to write the records to.
* @param start The logical start position of the subset.
- * @param len The number of elements to write.
+ * @param num The number of elements to write.
* @throws IOException Thrown, if an I/O exception occurred writing to the output view.
*/
public void writeToOutput(final ChannelWriterOutputView output, final int start, int num) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeIterator.java
index 9ef2ad7..6881cdf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeIterator.java
@@ -34,24 +34,19 @@ import org.apache.flink.util.MutableObjectIterator;
* to the next smallest element logarithmic in complexity, with respect to the
* number of streams to be merged.
* The order among the elements is established using the methods from the
- * {@link TypeSerializer} class, specifically {@link TypeSerializer#setReference(Object)}
- * and {@link TypeSerializer#compareToReference(TypeSerializer)}.
- *
- * @see TypeSerializer
- * @see TypeSerializer#setReference(Object)
- * @see TypeSerializer#compareToReference(TypeSerializer)
- *
+ * {@link TypeComparator} class, specifically {@link TypeComparator#setReference(Object)}
+ * and {@link TypeComparator#compareToReference(TypeComparator)}.
*/
-public class MergeIterator<E> implements MutableObjectIterator<E>
-{
+public class MergeIterator<E> implements MutableObjectIterator<E> {
+
private final PartialOrderPriorityQueue<HeadStream<E>> heap; // heap over the head elements of the stream
private final TypeSerializer<E> serializer;
/**
* @param iterators
- * @param accessors The accessors used to establish an order among the elements.
- * The accessors will not be used directly, but a duplicate will be used.
+ * @param serializer
+ * @param comparator
* @throws IOException
*/
public MergeIterator(List<MutableObjectIterator<E>> iterators,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
index 2ed75ae..48693a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/MergeMatchIterator.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.sort;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIterator.java
index dcd3361..f6803fc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIterator.java
@@ -16,17 +16,15 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.sort;
import java.io.IOException;
-import java.util.Iterator;
+import java.util.Collections;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.operators.util.CoGroupTaskIterator;
-import org.apache.flink.runtime.util.EmptyIterator;
import org.apache.flink.runtime.util.KeyGroupedIterator;
import org.apache.flink.util.MutableObjectIterator;
@@ -41,9 +39,9 @@ public class SortMergeCoGroupIterator<T1, T2> implements CoGroupTaskIterator<T1,
private MatchStatus matchStatus;
- private Iterator<T1> firstReturn;
+ private Iterable<T1> firstReturn;
- private Iterator<T2> secondReturn;
+ private Iterable<T2> secondReturn;
private TypePairComparator<T1, T2> comp;
@@ -73,13 +71,13 @@ public class SortMergeCoGroupIterator<T1, T2> implements CoGroupTaskIterator<T1,
@Override
- public Iterator<T1> getValues1() {
+ public Iterable<T1> getValues1() {
return this.firstReturn;
}
@Override
- public Iterator<T2> getValues2() {
+ public Iterable<T2> getValues2() {
return this.secondReturn;
}
@@ -117,7 +115,7 @@ public class SortMergeCoGroupIterator<T1, T2> implements CoGroupTaskIterator<T1,
}
else if (firstEmpty && !secondEmpty) {
// input1 is empty, input2 not
- this.firstReturn = EmptyIterator.get();
+ this.firstReturn = Collections.emptySet();
this.secondReturn = this.iterator2.getValues();
this.matchStatus = MatchStatus.FIRST_EMPTY;
return true;
@@ -125,7 +123,7 @@ public class SortMergeCoGroupIterator<T1, T2> implements CoGroupTaskIterator<T1,
else if (!firstEmpty && secondEmpty) {
// input1 is not empty, input 2 is empty
this.firstReturn = this.iterator1.getValues();
- this.secondReturn = EmptyIterator.get();
+ this.secondReturn = Collections.emptySet();
this.matchStatus = MatchStatus.SECOND_EMPTY;
return true;
}
@@ -142,12 +140,12 @@ public class SortMergeCoGroupIterator<T1, T2> implements CoGroupTaskIterator<T1,
else if (0 < comp) {
// key1 goes first
this.firstReturn = this.iterator1.getValues();
- this.secondReturn = EmptyIterator.get();
+ this.secondReturn = Collections.emptySet();
this.matchStatus = MatchStatus.SECOND_REMAINED;
}
else {
// key 2 goes first
- this.firstReturn = EmptyIterator.get();
+ this.firstReturn = Collections.emptySet();
this.secondReturn = this.iterator2.getValues();
this.matchStatus = MatchStatus.FIRST_REMAINED;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/CoGroupTaskIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/CoGroupTaskIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/CoGroupTaskIterator.java
index a20490f..e137e27 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/CoGroupTaskIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/CoGroupTaskIterator.java
@@ -20,7 +20,6 @@
package org.apache.flink.runtime.operators.util;
import java.io.IOException;
-import java.util.Iterator;
import org.apache.flink.runtime.memorymanager.MemoryAllocationException;
@@ -30,8 +29,8 @@ import org.apache.flink.runtime.memorymanager.MemoryAllocationException;
* @param <T1> The generic type of the first input's data type.
* @param <T2> The generic type of the second input's data type.
*/
-public interface CoGroupTaskIterator<T1, T2>
-{
+public interface CoGroupTaskIterator<T1, T2> {
+
/**
* General-purpose open method.
*
@@ -63,12 +62,12 @@ public interface CoGroupTaskIterator<T1, T2>
*
* @return an iterable over the left input values for the current key.
*/
- Iterator<T1> getValues1();
+ Iterable<T1> getValues1();
/**
* Returns an iterable over the left input values for the current key.
*
* @return an iterable over the left input values for the current key.
*/
- Iterator<T2> getValues2();
+ Iterable<T2> getValues2();
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/util/EmptyIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/EmptyIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/EmptyIterator.java
index b1628d7..caafaea 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/EmptyIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/EmptyIterator.java
@@ -25,7 +25,7 @@ import java.util.NoSuchElementException;
/**
* An empty iterator that never returns anything.
*/
-public final class EmptyIterator<E> implements Iterator<E> {
+public final class EmptyIterator<E> implements Iterator<E>, Iterable<E> {
/**
* The singleton instance.
@@ -38,9 +38,9 @@ public final class EmptyIterator<E> implements Iterator<E> {
* @param <E> The type of the objects (not) returned by the iterator.
* @return An instance of the iterator.
*/
- public static <E> Iterator<E> get() {
+ public static <E> EmptyIterator<E> get() {
@SuppressWarnings("unchecked")
- Iterator<E> iter = (Iterator<E>) INSTANCE;
+ EmptyIterator<E> iter = (EmptyIterator<E>) INSTANCE;
return iter;
}
@@ -73,4 +73,9 @@ public final class EmptyIterator<E> implements Iterator<E> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<E> iterator() {
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/util/KeyGroupedIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/KeyGroupedIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/KeyGroupedIterator.java
index 0873baf..be43cc8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/KeyGroupedIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/KeyGroupedIterator.java
@@ -26,6 +26,7 @@ import java.util.NoSuchElementException;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.util.MutableObjectIterator;
+import org.apache.flink.util.TraversableOnceException;
/**
* The KeyValueIterator returns a key and all values that belong to the key (share the same key).
@@ -98,6 +99,8 @@ public final class KeyGroupedIterator<E> {
return false;
}
}
+
+ this.valuesIterator.iteratorAvailable = true;
// Whole value-iterator was read and a new key is available.
if (this.lookAheadHasNext) {
@@ -151,19 +154,20 @@ public final class KeyGroupedIterator<E> {
// --------------------------------------------------------------------------------------------
- public final class ValuesIterator implements Iterator<E>
- {
+ public final class ValuesIterator implements Iterator<E>, Iterable<E> {
+
private final TypeSerializer<E> serializer = KeyGroupedIterator.this.serializer;
private final TypeComparator<E> comparator = KeyGroupedIterator.this.comparator;
private E staging = this.serializer.createInstance();
private boolean currentIsUnconsumed = false;
+ private boolean iteratorAvailable = true;
+
private ValuesIterator() {}
@Override
- public boolean hasNext()
- {
+ public boolean hasNext() {
if (KeyGroupedIterator.this.current == null || KeyGroupedIterator.this.lookAheadHasNext) {
return false;
}
@@ -221,5 +225,16 @@ public final class KeyGroupedIterator<E> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<E> iterator() {
+ if (iteratorAvailable) {
+ iteratorAvailable = false;
+ return this;
+ }
+ else {
+ throw new TraversableOnceException();
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/util/MutableToRegularIteratorWrapper.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/MutableToRegularIteratorWrapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/MutableToRegularIteratorWrapper.java
index e707a50..14e4ae8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/MutableToRegularIteratorWrapper.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/MutableToRegularIteratorWrapper.java
@@ -25,6 +25,7 @@ import java.util.NoSuchElementException;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.util.MutableObjectIterator;
+import org.apache.flink.util.TraversableOnceException;
/**
* This class wraps a {@link MutableObjectIterator} into a regular {@link Iterator}.
@@ -32,13 +33,15 @@ import org.apache.flink.util.MutableObjectIterator;
* whenever hasNext() returns (possibly with false), the previous obtained record is
* still valid and cannot have been overwritten internally.
*/
-public class MutableToRegularIteratorWrapper<T> implements Iterator<T> {
+public class MutableToRegularIteratorWrapper<T> implements Iterator<T>, Iterable<T> {
private final MutableObjectIterator<T> source;
private T current, next;
private boolean currentIsAvailable;
+
+ private boolean iteratorAvailable = true;
public MutableToRegularIteratorWrapper(MutableObjectIterator<T> source, TypeSerializer<T> serializer) {
this.source = source;
@@ -85,4 +88,15 @@ public class MutableToRegularIteratorWrapper<T> implements Iterator<T> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<T> iterator() {
+ if (iteratorAvailable) {
+ iteratorAvailable = false;
+ return this;
+ }
+ else {
+ throw new TraversableOnceException();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/main/java/org/apache/flink/runtime/util/SingleElementIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SingleElementIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SingleElementIterator.java
index 222c5b8..f7eb41a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SingleElementIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SingleElementIterator.java
@@ -27,7 +27,7 @@ import java.util.NoSuchElementException;
*
* @param<E> The generic type of the iterator.
*/
-public final class SingleElementIterator<E> implements Iterator<E> {
+public final class SingleElementIterator<E> implements Iterator<E>, Iterable<E> {
private E current;
private boolean available = false;
@@ -61,4 +61,9 @@ public final class SingleElementIterator<E> implements Iterator<E> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<E> iterator() {
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
index 3894233..3c8bbd0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CachedMatchTaskTest.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
import java.util.ArrayList;
@@ -24,9 +23,12 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.flink.api.common.functions.FlatJoinFunction;
-import org.apache.flink.api.java.record.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
+import org.apache.flink.runtime.operators.BuildFirstCachedMatchDriver;
+import org.apache.flink.runtime.operators.BuildSecondCachedMatchDriver;
+import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
@@ -40,8 +42,8 @@ import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
-public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record, Record, Record>>
-{
+public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record, Record, Record>> {
+
private static final long HASH_MEM = 6*1024*1024;
private static final long SORT_MEM = 3*1024*1024;
@@ -459,7 +461,7 @@ public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record,
// =================================================================================================
- public static final class MockMatchStub extends JoinFunction {
+ public static final class MockMatchStub extends RichFlatJoinFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
@Override
@@ -468,13 +470,13 @@ public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record,
}
}
- public static final class MockFailingMatchStub extends JoinFunction {
+ public static final class MockFailingMatchStub extends RichFlatJoinFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@Override
- public void join(Record record1, Record record2, Collector<Record> out) {
+ public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
if (++this.cnt >= 10) {
throw new ExpectedTestException();
}
@@ -483,7 +485,7 @@ public class CachedMatchTaskTest extends DriverTestBase<FlatJoinFunction<Record,
}
}
- public static final class MockDelayingMatchStub extends JoinFunction {
+ public static final class MockDelayingMatchStub extends RichFlatJoinFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
index 5551485..b65f161 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskExternalITCase.java
@@ -16,16 +16,15 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
-import java.util.Iterator;
-
import org.junit.Assert;
-
import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
+import org.apache.flink.runtime.operators.CoGroupDriver;
+import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
import org.apache.flink.types.IntValue;
@@ -84,25 +83,23 @@ public class CoGroupTaskExternalITCase extends DriverTestBase<CoGroupFunction<Re
Assert.assertEquals("Wrong result set size.", expCnt, this.output.getNumberOfRecords());
}
- public static final class MockCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
+ public static final class MockCoGroupStub extends RichCoGroupFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
private final Record res = new Record();
+ @SuppressWarnings("unused")
@Override
- public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out)
- {
+ public void coGroup(Iterable<Record> records1, Iterable<Record> records2, Collector<Record> out) {
int val1Cnt = 0;
int val2Cnt = 0;
- while (records1.hasNext()) {
+ for (Record r : records1) {
val1Cnt++;
- records1.next();
}
- while (records2.hasNext()) {
+ for (Record r : records2) {
val2Cnt++;
- records2.next();
}
if (val1Cnt == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
index e57d20c..968d947 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
@@ -16,17 +16,17 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
-import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-
import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
+import org.apache.flink.runtime.operators.CoGroupDriver;
+import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.CoGroupTaskExternalITCase.MockCoGroupStub;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -400,24 +400,20 @@ public class CoGroupTaskTest extends DriverTestBase<CoGroupFunction<Record, Reco
Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}
- public static class MockFailingCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
+ public static class MockFailingCoGroupStub extends RichCoGroupFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@Override
- public void coGroup(Iterator<Record> records1,
- Iterator<Record> records2, Collector<Record> out) throws RuntimeException
- {
+ public void coGroup(Iterable<Record> records1, Iterable<Record> records2, Collector<Record> out) {
int val1Cnt = 0;
- while (records1.hasNext()) {
+ for (@SuppressWarnings("unused") Record r : records1) {
val1Cnt++;
- records1.next();
}
- while (records2.hasNext()) {
- Record record2 = records2.next();
+ for (Record record2 : records2) {
if (val1Cnt == 0) {
if(++this.cnt>=10) {
@@ -440,25 +436,23 @@ public class CoGroupTaskTest extends DriverTestBase<CoGroupFunction<Record, Reco
}
- public static final class MockDelayingCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
+ public static final class MockDelayingCoGroupStub extends RichCoGroupFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
+ @SuppressWarnings("unused")
@Override
- public void coGroup(Iterator<Record> records1,
- Iterator<Record> records2, Collector<Record> out) {
+ public void coGroup(Iterable<Record> records1, Iterable<Record> records2, Collector<Record> out) {
- while (records1.hasNext()) {
+ for (Record r : records1) {
try {
Thread.sleep(100);
} catch (InterruptedException e) { }
- records1.next();
}
- while (records2.hasNext()) {
+ for (Record r : records2) {
try {
Thread.sleep(100);
} catch (InterruptedException e) { }
- records2.next();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
index 99440aa..7915d1f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
@@ -16,19 +16,18 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.GroupReduceCombineDriver;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -163,18 +162,18 @@ public class CombineTaskTest extends DriverTestBase<RichGroupReduceFunction<Reco
}
@Combinable
- public static class MockCombiningReduceStub extends ReduceFunction {
+ public static class MockCombiningReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue theInteger = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.theInteger);
sum += this.theInteger.getValue();
@@ -185,13 +184,13 @@ public class CombineTaskTest extends DriverTestBase<RichGroupReduceFunction<Reco
}
@Override
- public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
+ public void combine(Iterable<Record> records, Collector<Record> out) throws Exception {
reduce(records, out);
}
}
@Combinable
- public static final class MockFailingCombiningReduceStub extends ReduceFunction {
+ public static final class MockFailingCombiningReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@@ -201,12 +200,12 @@ public class CombineTaskTest extends DriverTestBase<RichGroupReduceFunction<Reco
private final IntValue combineValue = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.value);
sum += this.value.getValue();
@@ -218,12 +217,12 @@ public class CombineTaskTest extends DriverTestBase<RichGroupReduceFunction<Reco
}
@Override
- public void combine(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void combine(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.combineValue);
sum += this.combineValue.getValue();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MapTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MapTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MapTaskTest.java
index b7e48c6..786fb4e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MapTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MapTaskTest.java
@@ -37,6 +37,7 @@ import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
+@SuppressWarnings("deprecation")
public class MapTaskTest extends DriverTestBase<GenericCollectorMap<Record, Record>> {
private static final Log LOG = LogFactory.getLog(MapTaskTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
index ebee0b8..8b8e991 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.junit.Assert;
@@ -28,10 +26,11 @@ import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.GroupReduceDriver;
import org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
@@ -213,19 +212,19 @@ public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunc
}
- public static class MockReduceStub extends ReduceFunction {
+ public static class MockReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue key = new IntValue();
private final IntValue value = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int cnt = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
cnt++;
}
element.getField(0, this.key);
@@ -236,7 +235,7 @@ public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunc
}
@Combinable
- public static class MockCombiningReduceStub extends ReduceFunction {
+ public static class MockCombiningReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue key = new IntValue();
@@ -244,12 +243,12 @@ public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunc
private final IntValue combineValue = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.value);
sum += this.value.getValue();
@@ -261,12 +260,12 @@ public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunc
}
@Override
- public void combine(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void combine(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.combineValue);
sum += this.combineValue.getValue();
@@ -276,7 +275,5 @@ public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunc
element.setField(1, this.combineValue);
out.collect(element);
}
-
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
index b367a7e..fada5a4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -29,10 +27,11 @@ import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
+import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.runtime.operators.GroupReduceDriver;
import org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -270,19 +269,19 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
}
- public static class MockReduceStub extends ReduceFunction {
+ public static class MockReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue key = new IntValue();
private final IntValue value = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int cnt = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
cnt++;
}
element.getField(0, this.key);
@@ -293,7 +292,7 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
}
@Combinable
- public static class MockCombiningReduceStub extends ReduceFunction {
+ public static class MockCombiningReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue key = new IntValue();
@@ -301,12 +300,12 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
private final IntValue combineValue = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.value);
sum += this.value.getValue();
@@ -318,12 +317,12 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
}
@Override
- public void combine(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void combine(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
element.getField(1, this.combineValue);
sum += this.combineValue.getValue();
@@ -336,7 +335,7 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
}
- public static class MockFailingReduceStub extends ReduceFunction {
+ public static class MockFailingReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@@ -345,12 +344,12 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
private final IntValue value = new IntValue();
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out)
- throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
Record element = null;
int valCnt = 0;
- while (records.hasNext()) {
- element = records.next();
+
+ for (Record next : records) {
+ element = next;
valCnt++;
}
@@ -365,16 +364,15 @@ public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Recor
}
}
- public static class MockDelayingReduceStub extends ReduceFunction {
+ public static class MockDelayingReduceStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out) {
- while(records.hasNext()) {
+ public void reduce(Iterable<Record> records, Collector<Record> out) {
+ for (@SuppressWarnings("unused") Record r : records) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {}
- records.next();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/chaining/ChainTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/chaining/ChainTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/chaining/ChainTaskTest.java
index aabaa03..5609128 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/chaining/ChainTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/chaining/ChainTaskTest.java
@@ -16,16 +16,14 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.chaining;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.functions.GenericCollectorMap;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
import org.apache.flink.configuration.Configuration;
@@ -48,6 +46,7 @@ import org.junit.Assert;
import org.junit.Test;
+@SuppressWarnings("deprecation")
public class ChainTaskTest extends TaskTestBase {
private static final int MEMORY_MANAGER_SIZE = 1024 * 1024 * 3;
@@ -68,7 +67,6 @@ public class ChainTaskTest extends TaskTestBase {
}
-
@Test
public void testMapTask() {
final int keyCnt = 100;
@@ -190,18 +188,19 @@ public class ChainTaskTest extends TaskTestBase {
}
}
- public static final class MockFailingCombineStub extends ReduceFunction {
+ public static final class MockFailingCombineStub extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
+ public void reduce(Iterable<Record> records, Collector<Record> out) throws Exception {
if (++this.cnt >= 5) {
throw new RuntimeException("Expected Test Exception");
}
- while(records.hasNext()) {
- out.collect(records.next());
+
+ for (Record r : records) {
+ out.collect(r);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
index d603dec..427dc74 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.drivers;
import java.util.Arrays;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.functions.GroupReduceFunction;
@@ -155,11 +153,10 @@ public class AllGroupReduceDriverTest {
public static final class ConcatSumReducer extends RichGroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
@Override
- public void reduce(Iterator<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
- Tuple2<String, Integer> current = values.next();
+ public void reduce(Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
+ Tuple2<String, Integer> current = new Tuple2<String, Integer>("", 0);
- while (values.hasNext()) {
- Tuple2<String, Integer> next = values.next();
+ for (Tuple2<String, Integer> next : values) {
next.f0 = current.f0 + next.f0;
next.f1 = current.f1 + next.f1;
current = next;
@@ -172,11 +169,10 @@ public class AllGroupReduceDriverTest {
public static final class ConcatSumMutableReducer extends RichGroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
@Override
- public void reduce(Iterator<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
- Tuple2<StringValue, IntValue> current = values.next();
+ public void reduce(Iterable<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
+ Tuple2<StringValue, IntValue> current = new Tuple2<StringValue, IntValue>(new StringValue(""), new IntValue(0));
- while (values.hasNext()) {
- Tuple2<StringValue, IntValue> next = values.next();
+ for (Tuple2<StringValue, IntValue> next : values) {
next.f0.append(current.f0);
next.f1.setValue(current.f1.getValue() + next.f1.getValue());
current = next;
[24/39] [FLINK-701] Several cleanups after SAM refactoring. - Lambda
detection compiles on earlier java versions - Add lambda detection test. -
Fix JavaDocs
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
index 2293b5e..c045508 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatMapFunction.java
@@ -20,22 +20,14 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
/**
- * The abstract base class for flatMap functions. FlatMap functions take elements and transform them,
- * into zero, one, or more elements. Typical applications can be splitting elements, or unnesting lists
- * and arrays. Operations that produce multiple strictly one result element per input element can also
- * use the {@link RichMapFunction}.
- * <p>
- * The basic syntax for using a FlatMapFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<Y> result = input.flatMap(new MyFlatMapFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the FlatMapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link FlatMapFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN> Type of the input elements.
* @param <OUT> Type of the returned elements.
@@ -44,16 +36,6 @@ public abstract class RichFlatMapFunction<IN, OUT> extends AbstractRichFunction
private static final long serialVersionUID = 1L;
- /**
- * The core method of the FlatMapFunction. Takes an element from the input data set and transforms
- * it into zero, one, or more elements.
- *
- * @param value The input value.
- * @param out The collector for for emitting result values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract void flatMap(IN value, Collector<OUT> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
index eb75f53..801f592 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
@@ -27,26 +27,14 @@ import java.util.Iterator;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
/**
- * The abstract base class for group reduce functions. Group reduce functions process groups of elements.
- * They may aggregate them to a single value, or produce multiple result values for each group.
- * <p>
- * For a reduce functions that works incrementally by combining always two elements, see
- * {@link RichReduceFunction}, called via {@link org.apache.flink.api.java.DataSet#reduce(RichReduceFunction)}.
- * <p>
- * The basic syntax for using a grouped GroupReduceFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.groupBy(<key-definition>).reduceGroup(new MyGroupReduceFunction());
- * </blockquote></pre>
- * <p>
- * GroupReduceFunctions may be "combinable", in which case they can pre-reduce partial groups in order to
- * reduce the data volume early. See the {@link #combine(Iterator, Collector)} function for details.
- * <p>
- * Like all functions, the GroupReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link GroupReduceFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN> Type of the elements that this function processes.
* @param <OUT> The type of the elements returned by the user-defined function.
@@ -55,16 +43,6 @@ public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunct
private static final long serialVersionUID = 1L;
- /**
- * Core method of the reduce function. It is called one per group of elements. If the reducer
- * is not grouped, than the entire data set is considered one group.
- *
- * @param values The iterator returning the group of values to be reduced.
- * @param out The collector to emit the returned values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract void reduce(Iterator<IN> values, Collector<OUT> out) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
index 7eaf44c..a0c28ee 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichJoinFunction.java
@@ -18,10 +18,20 @@
package org.apache.flink.api.java.functions;
-
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+/**
+ * Rich variant of the {@link JoinFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
+ *
+ * @param <IN1> The type of the elements in the first input.
+ * @param <IN2> The type of the elements in the second input.
+ * @param <OUT> The type of the result elements.
+ */
public abstract class RichJoinFunction<IN1,IN2,OUT> extends AbstractRichFunction implements JoinFunction<IN1,IN2,OUT> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
index 54de7d4..f6f5356 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichMapFunction.java
@@ -20,22 +20,13 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichFunction;
/**
- * The abstract base class for Map functions. Map functions take elements and transform them,
- * element wise. A Map function always produces a single result element for each input element.
- * Typical applications are parsing elements, converting data types, or projecting out fields.
- * Operations that produce multiple result elements from a single input element can be implemented
- * using the {@link RichFlatMapFunction}.
- * <p>
- * The basic syntax for using a MapFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<Y> result = input.map(new MyMapFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the MapFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link MapFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <IN> Type of the input elements.
* @param <OUT> Type of the returned elements.
@@ -44,16 +35,6 @@ public abstract class RichMapFunction<IN, OUT> extends AbstractRichFunction impl
private static final long serialVersionUID = 1L;
- /**
- * The core method of the MapFunction. Takes an element from the input data set and transforms
- * it into another element.
- *
- * @param value The input value.
- * @return The value produced by the map function from the input value.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
@Override
public abstract OUT map(IN value) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
index 35cb392..a63f8dc 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichReduceFunction.java
@@ -20,27 +20,13 @@ package org.apache.flink.api.java.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichFunction;
/**
- * The abstract base class for Reduce functions. Reduce functions combine groups of elements to
- * a single value, by taking always two elements and combining them into one. Reduce functions
- * may be used on entire data sets, or on grouped data sets. In the latter case, each group is reduced
- * individually.
- * <p>
- * For a reduce functions that work on an entire group at the same time (such as the
- * MapReduce/Hadoop-style reduce), see {@link RichGroupReduceFunction}, called via
- * {@link org.apache.flink.api.java.DataSet#reduceGroup(RichGroupReduceFunction)}. In the general case,
- * ReduceFunctions are considered faster, because they allow the system to use hash-based
- * execution strategies.
- * <p>
- * The basic syntax for using a grouped ReduceFunction is as follows:
- * <pre><blockquote>
- * DataSet<X> input = ...;
- *
- * DataSet<X> result = input.groupBy(<key-definition>).reduce(new MyReduceFunction());
- * </blockquote></pre>
- * <p>
- * Like all functions, the ReduceFunction needs to be serializable, as defined in {@link java.io.Serializable}.
+ * Rich variant of the {@link ReduceFunction}. As a {@link RichFunction}, it gives access to the
+ * {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
+ * {@link RichFunction#open(org.apache.flink.configuration.Configuration)} and
+ * {@link RichFunction#close()}.
*
* @param <T> Type of the elements that this function processes.
*/
@@ -48,16 +34,5 @@ public abstract class RichReduceFunction<T> extends AbstractRichFunction impleme
private static final long serialVersionUID = 1L;
- /**
- * The core method of the ReduceFunction, combining two values into one value of the same type.
- * The reduce function is consecutively applied to all values of a group until only a single value remains.
- *
- * @param value1 The first value to combine.
- * @param value2 The second value to combine.
- * @return The combined value of both input values.
- *
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
- * to fail and may trigger recovery.
- */
public abstract T reduce(T value1, T value2) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
index 89c3334..744893b 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
@@ -515,7 +515,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
if (function == null) {
throw new NullPointerException("CoGroup function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ if (FunctionUtils.isLambdaFunction(function)) {
throw new UnsupportedLambdaExpressionException();
}
TypeInformation<R> returnType = TypeExtractor.getCoGroupReturnTypes(function, input1.getType(), input2.getType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
index d1e99d6..a24a093 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CrossOperator.java
@@ -134,7 +134,7 @@ public class CrossOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OUT,
if (function == null) {
throw new NullPointerException("Cross function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ if (FunctionUtils.isLambdaFunction(function)) {
throw new UnsupportedLambdaExpressionException();
}
TypeInformation<R> returnType = TypeExtractor.getCrossReturnTypes(function, input1.getType(), input2.getType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
index 591551f..7646fa0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
@@ -21,7 +21,6 @@ package org.apache.flink.api.java.operators;
import java.util.Iterator;
import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Operator;
@@ -29,6 +28,7 @@ import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -80,7 +80,6 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
protected org.apache.flink.api.common.operators.base.GroupReduceOperatorBase<?, T, ?> translateToDataFlow(Operator<T> input) {
final RichGroupReduceFunction<T, T> function = new DistinctFunction<T>();
- final FlatCombineFunction<T> combineFunction = new DistinctCombiner<T>();
String name = function.getClass().getName();
@@ -104,7 +103,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
PlanUnwrappingReduceGroupOperator<T, T, ?> po = translateSelectorFunctionDistinct(
- selectorKeys, function, combineFunction, getInputType(), getResultType(), name, input, true);
+ selectorKeys, function, getInputType(), getResultType(), name, input);
po.setDegreeOfParallelism(this.getParallelism());
@@ -118,9 +117,8 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
// --------------------------------------------------------------------------------------------
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionDistinct(
- Keys.SelectorFunctionKeys<IN, ?> rawKeys, RichGroupReduceFunction<IN, OUT> function, FlatCombineFunction<IN> combineFunction,
- TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input,
- boolean combinable)
+ Keys.SelectorFunctionKeys<IN, ?> rawKeys, RichGroupReduceFunction<IN, OUT> function,
+ TypeInformation<IN> inputType, TypeInformation<OUT> outputType, String name, Operator<IN> input)
{
@SuppressWarnings("unchecked")
final Keys.SelectorFunctionKeys<IN, K> keys = (Keys.SelectorFunctionKeys<IN, K>) rawKeys;
@@ -131,7 +129,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
- new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, combinable);
+ new PlanUnwrappingReduceGroupOperator<IN, OUT, K>(function, keys, name, outputType, typeInfoWithKey, true);
MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>> mapper = new MapOperatorBase<IN, Tuple2<K, IN>, MapFunction<IN, Tuple2<K, IN>>>(extractor, new UnaryOperatorInformation<IN, Tuple2<K, IN>>(inputType, typeInfoWithKey), "Key Extractor");
@@ -144,26 +142,14 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
return reducer;
}
+ @Combinable
public static final class DistinctFunction<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<T> values, Collector<T> out)
- throws Exception {
+ public void reduce(Iterator<T> values, Collector<T> out) {
out.collect(values.next());
}
}
-
- public static final class DistinctCombiner<T> implements FlatCombineFunction<T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void combine(Iterator<T> values, Collector<T> out)
- throws Exception {
- out.collect(values.next());
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
index 7ab0b11..e1424ad 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
@@ -51,9 +51,6 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
private final Grouping<IN> grouper;
- // reduceFunction is a GroupReduceFunction
- private boolean richFunction;
-
private boolean combinable;
/**
@@ -176,8 +173,8 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
- new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
+ GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
+ new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
po.setCombinable(combinable);
po.setInput(input);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
index 3223f4d..200c4de 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Grouping.java
@@ -26,8 +26,8 @@ import org.apache.flink.api.java.DataSet;
* Grouping is an intermediate step for a transformation on a grouped DataSet.<br/>
* The following transformation can be applied on Grouping:
* <ul>
- * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.java.functions.RichReduceFunction)},</li>
- * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)}, and</li>
+ * <li>{@link UnsortedGrouping#reduce(org.apache.flink.api.common.functions.ReduceFunction)},</li>
+ * <li>{@link UnsortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)}, and</li>
* <li>{@link UnsortedGrouping#aggregate(org.apache.flink.api.java.aggregation.Aggregations, int)}.</li>
* </ul>
*
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
index a07a157..ce0aea7 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
@@ -467,7 +467,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
if (function == null) {
throw new NullPointerException("Join function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ if (FunctionUtils.isLambdaFunction(function)) {
throw new UnsupportedLambdaExpressionException();
}
TypeInformation<R> returnType = TypeExtractor.getJoinReturnTypes(function, getInput1Type(), getInput2Type());
@@ -478,10 +478,10 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
if (function == null) {
throw new NullPointerException("Join function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(function)) {
+ if (FunctionUtils.isLambdaFunction(function)) {
throw new UnsupportedLambdaExpressionException();
}
- FlatJoinFunction generatedFunction = new WrappingFlatJoinFunction<I1, I2, R>(function);
+ FlatJoinFunction<I1, I2, R> generatedFunction = new WrappingFlatJoinFunction<I1, I2, R>(function);
TypeInformation<R> returnType = TypeExtractor.getJoinReturnTypes(function, getInput1Type(), getInput2Type());
return new EquiJoin<I1, I2, R>(getInput1(), getInput2(), getKeys1(), getKeys2(), generatedFunction, function, returnType, getJoinHint());
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
index 97b2417..767f75a 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
@@ -32,7 +32,7 @@ import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
* SortedGrouping is an intermediate step for a transformation on a grouped and sorted DataSet.<br/>
* The following transformation can be applied on sorted groups:
* <ul>
- * <li>{@link SortedGrouping#reduceGroup(org.apache.flink.api.java.functions.RichGroupReduceFunction)},</li>
+ * <li>{@link SortedGrouping#reduceGroup(org.apache.flink.api.common.functions.GroupReduceFunction)},</li>
* </ul>
*
* @param <T> The type of the elements of the sorted and grouped DataSet.
@@ -82,7 +82,7 @@ public class SortedGrouping<T> extends Grouping<T> {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ if (FunctionUtils.isLambdaFunction(reducer)) {
throw new UnsupportedLambdaExpressionException();
}
return new GroupReduceOperator<T, R>(this, reducer);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
index 9e71ba0..87b1454 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
@@ -127,7 +127,7 @@ public class UnsortedGrouping<T> extends Grouping<T> {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.");
}
- if (FunctionUtils.isSerializedLambdaFunction(reducer)) {
+ if (FunctionUtils.isLambdaFunction(reducer)) {
throw new UnsupportedLambdaExpressionException();
}
return new GroupReduceOperator<T, R>(this, reducer);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
index 5e80455..29eb5ed 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
@@ -39,7 +39,7 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
public PlanUnwrappingReduceGroupOperator(GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name,
TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable)
{
- super(combinable ? new TupleUnwrappingFlatCombinableGroupReducer<IN, OUT, K>((RichGroupReduceFunction) udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf),
+ super(combinable ? new TupleUnwrappingFlatCombinableGroupReducer<IN, OUT, K>((RichGroupReduceFunction<IN, OUT>) udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf),
new UnaryOperatorInformation<Tuple2<K, IN>, OUT>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name);
super.setCombinable(combinable);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
index fa0ca11..e9b5c25 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
@@ -26,7 +26,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
/**
- * The CoGroupFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.operators.CoGroupOperator}.
+ * The CoGroupFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.record.operators.CoGroupOperator}.
*/
public abstract class CoGroupFunction extends AbstractRichFunction implements org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
index c4587fd..3a6c931 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CrossFunction.java
@@ -23,7 +23,7 @@ import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.types.Record;
/**
- * The CrossFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.operators.CrossOperator}.
+ * The CrossFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.record.operators.CrossOperator}.
*/
public abstract class CrossFunction extends AbstractRichFunction implements org.apache.flink.api.common.functions.CrossFunction<Record, Record, Record> {
@@ -41,10 +41,6 @@ public abstract class CrossFunction extends AbstractRichFunction implements org.
* runtime catches an exception, it aborts the task and lets the fail-over logic
* decide whether to retry the task execution.
*/
-
- //@Override
- //public abstract void cross(Record record1, Record record2, Collector<Record> out) throws Exception;
-
@Override
public abstract Record cross(Record first, Record second) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
index dce24a3..cc4f96b 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/JoinFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.java.record.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
@@ -25,25 +24,13 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
/**
- * The JoinFunction must implementation by functions of a {@link org.apache.flink.api.java.operators.JoinOperator}.
+ * The JoinFunction must implementation by functions of a {@link org.apache.flink.api.java.record.operators.JoinOperator}.
* It resembles an equality join of both inputs on their key fields.
*/
public abstract class JoinFunction extends AbstractRichFunction implements FlatJoinFunction<Record, Record, Record> {
private static final long serialVersionUID = 1L;
- /**
- * This method must be implemented to provide a user implementation of a join.
- * It is called for each two records that share the same key and come from different inputs.
- *
- * @param value1 The record that comes from the first input.
- * @param value2 The record that comes from the second input.
- * @return The result of the join UDF as record
- *
- * @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
- * runtime catches an exception, it aborts the combine task and lets the fail-over logic
- * decide whether to retry the combiner execution.
- */
@Override
public abstract void join(Record value1, Record value2, Collector<Record> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
index 99c945d..b082e2d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/MapFunction.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.java.record.functions;
import org.apache.flink.api.common.functions.AbstractRichFunction;
@@ -28,6 +27,7 @@ import org.apache.flink.util.Collector;
* The MapFunction must be extended to provide a mapper implementation
* By definition, the mapper is called for each individual input record.
*/
+@SuppressWarnings("deprecation")
public abstract class MapFunction extends AbstractRichFunction implements GenericCollectorMap<Record, Record> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
index 073b11a..a1e6369 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
@@ -29,7 +29,7 @@ import org.apache.flink.util.Collector;
/**
* The ReduceFunction must be extended to provide a reducer implementation, as invoked by a
- * {@link org.apache.flink.api.java.operators.ReduceOperator}.
+ * {@link org.apache.flink.api.java.record.operators.ReduceOperator}.
*/
public abstract class ReduceFunction extends AbstractRichFunction implements GroupReduceFunction<Record, Record>, FlatCombineFunction<Record> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/main/java/org/apache/flink/api/java/record/operators/MapOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/MapOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/MapOperator.java
index 64f70f6..85afa64 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/MapOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/MapOperator.java
@@ -41,6 +41,7 @@ import org.apache.flink.types.Record;
*
* @see MapFunction
*/
+@SuppressWarnings("deprecation")
public class MapOperator extends CollectorMapOperatorBase<Record, Record, MapFunction> implements RecordOperator {
private static String DEFAULT_NAME = "<Unnamed Mapper>";
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
index 155bbd1..be872e5 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
@@ -129,7 +129,7 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
assertEquals(IdentityMapper.class, worksetMapper.getUserCodeWrapper().getUserCodeClass());
assertEquals(NextWorksetMapper.class, nextWorksetMapper.getUserCodeWrapper().getUserCodeClass());
if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) {
- WrappingFunction wf = (WrappingFunction) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
+ WrappingFunction<?> wf = (WrappingFunction<?>) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DistrinctTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DistrinctTranslationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DistrinctTranslationTest.java
new file mode 100644
index 0000000..ec3898e
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DistrinctTranslationTest.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.operators.translation;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.DistinctOperator;
+import org.junit.Assert;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class DistrinctTranslationTest {
+
+ @Test
+ public void testCombinable() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> input = env.fromElements("1", "2", "1", "3");
+
+
+ DistinctOperator<String> op = input.distinct(new KeySelector<String, String>() {
+ public String getKey(String value) { return value; }
+ });
+
+ op.print();
+
+ Plan p = env.createProgramPlan();
+
+ GroupReduceOperatorBase<?, ?, ?> reduceOp = (GroupReduceOperatorBase<?, ?, ?>) p.getDataSinks().iterator().next().getInput();
+ Assert.assertTrue(reduceOp.isCombinable());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index c6ad73d..8346d00 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -1385,6 +1385,8 @@ public class TypeExtractorTest {
public void testFunction() {
RichMapFunction<String, Boolean> mapInterface = new RichMapFunction<String, Boolean>() {
+ private static final long serialVersionUID = 1L;
+
@Override
public void setRuntimeContext(RuntimeContext t) {
@@ -1417,6 +1419,8 @@ public class TypeExtractorTest {
@Test
public void testInterface() {
MapFunction<String, Boolean> mapInterface = new MapFunction<String, Boolean>() {
+ private static final long serialVersionUID = 1L;
+
@Override
public Boolean map(String record) throws Exception {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
index c417249..7dd4dea 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class CoGroupITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
index f8d217e..3875bab 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class CrossITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
index c775425..bb04336 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
@@ -23,8 +23,6 @@ import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -34,9 +32,11 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
-@RunWith(Parameterized.class)
+@SuppressWarnings("serial")
public class FilterITCase extends JavaProgramTestBase {
+ private static final String EXPECTED_RESULT = "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n";
public static DataSet<Tuple3<Integer, Long, String>> get3TupleDataSet(ExecutionEnvironment env) {
@@ -68,15 +68,7 @@ public class FilterITCase extends JavaProgramTestBase {
return env.fromCollection(data);
}
- private static int NUM_PROGRAMS = 1;
-
- private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
- private String expectedResult;
-
- public FilterITCase(Configuration config) {
- super(config);
- }
@Override
protected void preSubmit() throws Exception {
@@ -85,58 +77,18 @@ public class FilterITCase extends JavaProgramTestBase {
@Override
protected void testProgram() throws Exception {
- expectedResult = FilterProgs.runProgram(curProgId, resultPath);
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
+ filter(value -> value.f2.contains("world"));
+ filterDs.writeAsCsv(resultPath);
+ env.execute();
}
@Override
protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(expectedResult, resultPath);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
-
- LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
-
- for(int i=1; i <= NUM_PROGRAMS; i++) {
- Configuration config = new Configuration();
- config.setInteger("ProgramId", i);
- tConfigs.add(config);
- }
-
- return toParameterList(tConfigs);
+ compareResultsByLinesInMemory(EXPECTED_RESULT, resultPath);
}
-
- private static class FilterProgs {
-
- public static String runProgram(int progId, String resultPath) throws Exception {
-
- switch(progId) {
- case 1: {
- /*
- * Test lambda filter
- * Functionality identical to org.apache.flink.test.javaApiOperators.FilterITCase test 3
- */
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
- filter(value -> value.f2.contains("world"));
- filterDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "3,2,Hello world\n" +
- "4,3,Hello world, how are you?\n";
- }
- default:
- throw new IllegalArgumentException("Invalid program id");
- }
-
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
index 043b4e8..431151e 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class FlatJoinITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
index 55f507c..5cf7fc2 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class FlatMapITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
index 494aff6..a86de1f 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class GroupReduceITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
index 3f4f696..d44d116 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class JoinITCase implements Serializable {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/LambdaExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/LambdaExtractionTest.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/LambdaExtractionTest.java
new file mode 100644
index 0000000..4c8ee23
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/LambdaExtractionTest.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+@SuppressWarnings("serial")
+public class LambdaExtractionTest {
+
+ @Test
+ public void testIdentifyLambdas() {
+ try {
+ MapFunction<?, ?> anonymousFromInterface = new MapFunction<String, Integer>() {
+ @Override
+ public Integer map(String value) { return Integer.parseInt(value); }
+ };
+
+ MapFunction<?, ?> anonymousFromClass = new RichMapFunction<String, Integer>() {
+ @Override
+ public Integer map(String value) { return Integer.parseInt(value); }
+ };
+
+ MapFunction<?, ?> fromProperClass = new StaticMapper();
+
+ MapFunction<?, ?> fromDerived = new ToTuple<Integer>() {
+ @Override
+ public Tuple2<Integer, Long> map(Integer value) {
+ return new Tuple2<Integer, Long>(value, 1L);
+ }
+ };
+
+ MapFunction<String, Integer> lambda = (str) -> Integer.parseInt(str);
+
+ assertFalse(FunctionUtils.isLambdaFunction(anonymousFromInterface));
+ assertFalse(FunctionUtils.isLambdaFunction(anonymousFromClass));
+ assertFalse(FunctionUtils.isLambdaFunction(fromProperClass));
+ assertFalse(FunctionUtils.isLambdaFunction(fromDerived));
+ assertTrue(FunctionUtils.isLambdaFunction(lambda));
+ assertTrue(FunctionUtils.isLambdaFunction(STATIC_LAMBDA));
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ public static class StaticMapper implements MapFunction<String, Integer> {
+
+ @Override
+ public Integer map(String value) { return Integer.parseInt(value); }
+ }
+
+ public interface ToTuple<T> extends MapFunction<T, Tuple2<T, Long>> {
+
+ @Override
+ public Tuple2<T, Long> map(T value) throws Exception;
+ }
+
+ private static final MapFunction<String, Integer> STATIC_LAMBDA = (str) -> Integer.parseInt(str);
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
index 3af360b..5e9f732 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import java.io.Serializable;
+@SuppressWarnings("serial")
public class MapITCase implements Serializable{
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
index ab27fe4..1a34814 100644
--- a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
@@ -36,9 +36,20 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
-@RunWith(Parameterized.class)
+@SuppressWarnings("serial")
public class ReduceITCase extends JavaProgramTestBase {
+ private static final String EXPECTED_RESULT = "1,1,0,Hallo,1\n" +
+ "2,3,2,Hallo Welt wie,1\n" +
+ "2,2,1,Hallo Welt,2\n" +
+ "3,9,0,P-),2\n" +
+ "3,6,5,BCD,3\n" +
+ "4,17,0,P-),1\n" +
+ "4,17,0,P-),2\n" +
+ "5,11,10,GHI,1\n" +
+ "5,29,0,P-),2\n" +
+ "5,25,0,P-),3\n";
+
public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> get5TupleDataSet(ExecutionEnvironment env) {
List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<Tuple5<Integer, Long, Integer, String, Long>>();
@@ -71,17 +82,9 @@ public class ReduceITCase extends JavaProgramTestBase {
return env.fromCollection(data, type);
}
-
- private static int NUM_PROGRAMS = 1;
-
- private int curProgId = config.getInteger("ProgramId", -1);
+
private String resultPath;
- private String expectedResult;
-
- public ReduceITCase(Configuration config) {
- super(config);
- }
-
+
@Override
protected void preSubmit() throws Exception {
resultPath = getTempDirPath("result");
@@ -89,72 +92,23 @@ public class ReduceITCase extends JavaProgramTestBase {
@Override
protected void testProgram() throws Exception {
- expectedResult = ReduceProgs.runProgram(curProgId, resultPath);
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = get5TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds
+ .groupBy(4, 0)
+ .reduce((in1, in2) -> {
+ Tuple5<Integer, Long, Integer, String, Long> out = new Tuple5<Integer, Long, Integer, String, Long>();
+ out.setFields(in1.f0, in1.f1 + in2.f1, 0, "P-)", in1.f4);
+ return out;
+ });
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
}
@Override
protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(expectedResult, resultPath);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
-
- LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
-
- for(int i=1; i <= NUM_PROGRAMS; i++) {
- Configuration config = new Configuration();
- config.setInteger("ProgramId", i);
- tConfigs.add(config);
- }
-
- return toParameterList(tConfigs);
- }
-
- private static class ReduceProgs {
-
- public static String runProgram(int progId, String resultPath) throws Exception {
-
- switch(progId) {
- case 1: {
- /*
- * Test reduce with lambda
- * Functionality identical to org.apache.flink.test.javaApiOperators.ReduceITCase test 2
- */
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = get5TupleDataSet(env);
- DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds
- .groupBy(4, 0)
- .reduce((in1, in2) -> {
- Tuple5<Integer, Long, Integer, String, Long> out = new Tuple5<Integer, Long, Integer, String, Long>();
- out.setFields(in1.f0, in1.f1 + in2.f1, 0, "P-)", in1.f4);
- return out;
- });
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1,0,Hallo,1\n" +
- "2,3,2,Hallo Welt wie,1\n" +
- "2,2,1,Hallo Welt,2\n" +
- "3,9,0,P-),2\n" +
- "3,6,5,BCD,3\n" +
- "4,17,0,P-),1\n" +
- "4,17,0,P-),2\n" +
- "5,11,10,GHI,1\n" +
- "5,29,0,P-),2\n" +
- "5,25,0,P-),3\n";
- }
- default:
- throw new IllegalArgumentException("Invalid program id");
- }
-
- }
-
+ compareResultsByLinesInMemory(EXPECTED_RESULT, resultPath);
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
index 1db3524..9ff7181 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.iterative.io;
import java.io.IOException;
@@ -28,13 +27,12 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the solution set of a workset iteration.
* <p/>
- * The records are written to a {@link MutableHashTable} hash table to allow in-memory point updates.
+ * The records are written to a HashTable hash table to allow in-memory point updates.
* <p/>
* Assumption for fast updates: the build side iterator of the hash table is already positioned for the update. This
* is for example the case when a solution set update happens directly after a solution set join. If this assumption
* doesn't hold, use {@link SolutionSetUpdateOutputCollector}, which probes the hash table before updating.
- *
- * @see {SolutionSetUpdateOutputCollector}
+
*/
public class SolutionSetFastUpdateOutputCollector<T> implements Collector<T> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
index 17670f1..89789c35 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
@@ -28,7 +28,7 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the solution set of a workset iteration.
* <p/>
- * The records are written to a {@link MutableHashTable} hash table to allow in-memory point updates.
+ * The records are written to a HashTable hash table to allow in-memory point updates.
* <p/>
* Records will only be collected, if there is a match after probing the hash table. If the build side iterator is
* already positioned for the update, use {@link SolutionSetFastUpdateOutputCollector} to the save re-probing.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 34cd232..636c492 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -23,7 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.IterationRuntimeContext;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -54,7 +54,7 @@ import java.io.IOException;
/**
* The base class for all tasks able to participate in an iteration.
*/
-public abstract class AbstractIterativePactTask<S extends RichFunction, OT> extends RegularPactTask<S, OT>
+public abstract class AbstractIterativePactTask<S extends Function, OT> extends RegularPactTask<S, OT>
implements Terminable
{
private static final Log log = LogFactory.getLog(AbstractIterativePactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index 7a77cff..797bbb6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.iterative.task;
import java.io.IOException;
@@ -25,7 +24,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -57,7 +56,7 @@ import org.apache.flink.util.MutableObjectIterator;
* The head is responsible for coordinating an iteration and can run a
* {@link org.apache.flink.runtime.operators.PactDriver} inside. It will read
* the initial input and establish a {@link BlockingBackChannel} to the iteration's tail. After successfully processing
- * the input, it will send {@link EndOfSuperstepEvent} events to its outputs. It must also be connected to a
+ * the input, it will send EndOfSuperstep events to its outputs. It must also be connected to a
* synchronization task and after each superstep, it will wait
* until it receives an {@link AllWorkersDoneEvent} from the sync, which signals that all other heads have also finished
* their iteration. Starting with
@@ -75,7 +74,7 @@ import org.apache.flink.util.MutableObjectIterator;
* The type of the feed-back data set (bulk partial solution / workset). For bulk iterations, {@code Y} is the
* same as {@code X}
*/
-public class IterationHeadPactTask<X, Y, S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT> {
+public class IterationHeadPactTask<X, Y, S extends Function, OT> extends AbstractIterativePactTask<S, OT> {
private static final Log log = LogFactory.getLog(IterationHeadPactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index 2a8325c..c23eae1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -16,14 +16,13 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.iterative.task;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.runtime.io.network.api.BufferWriter;
import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
@@ -32,16 +31,16 @@ import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
import org.apache.flink.util.Collector;
/**
- * An intermediate iteration task, which runs a {@link PactDriver} inside.
+ * An intermediate iteration task, which runs a Driver}inside.
* <p/>
* It will propagate {@link EndOfSuperstepEvent}s and {@link TerminationEvent}s to it's connected tasks. Furthermore
* intermediate tasks can also update the iteration state, either the workset or the solution set.
* <p/>
* If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
- * a {@link BlockingBackChannel} for the workset -XOR- a {@link MutableHashTable} for the solution set. In this case
+ * a {@link BlockingBackChannel} for the workset -XOR- a eHashTable for the solution set. In this case
* this task must be scheduled on the same instance as the head.
*/
-public class IterationIntermediatePactTask<S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT> {
+public class IterationIntermediatePactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT> {
private static final Log log = LogFactory.getLog(IterationIntermediatePactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
index a06ef5d..c44f443 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
@@ -44,9 +44,9 @@ import org.apache.flink.types.Value;
import com.google.common.base.Preconditions;
/**
- * The task responsible for synchronizing all iteration heads, implemented as an {@link AbstractOutputTask}. This task
+ * The task responsible for synchronizing all iteration heads, implemented as an output task. This task
* will never see any data.
- * In each superstep, it simply waits until it has receiced a {@link WorkerDoneEvent} from each head and will send back
+ * In each superstep, it simply waits until it has received a {@link WorkerDoneEvent} from each head and will send back
* an {@link AllWorkersDoneEvent} to signal that the next superstep can begin.
*/
public class IterationSynchronizationSinkTask extends AbstractInvokable implements Terminable {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index 942e2f6..90d732c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -16,12 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.iterative.task;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
@@ -29,16 +28,16 @@ import org.apache.flink.runtime.operators.PactTaskContext;
import org.apache.flink.util.Collector;
/**
- * An iteration tail, which runs a {@link PactDriver} inside.
+ * An iteration tail, which runs a driver inside.
* <p/>
* If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
- * a {@link BlockingBackChannel} for the workset -OR- a {@link MutableHashTable} for the solution set. Therefore this
+ * a BackChannel for the workset -OR- a HashTable for the solution set. Therefore this
* task must be scheduled on the same instance as the head. It's also possible for the tail to update *both* the workset
* and the solution set.
* <p/>
* If there is a separate solution set tail, the iteration head has to make sure to wait for it to finish.
*/
-public class IterationTailPactTask<S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT>
+public class IterationTailPactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT>
implements PactTaskContext<S, OT> {
private static final Log log = LogFactory.getLog(IterationTailPactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RuntimeExecutionContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RuntimeExecutionContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RuntimeExecutionContext.java
deleted file mode 100644
index 78cf1f5..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RuntimeExecutionContext.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.operators;
-
-import org.apache.flink.api.common.functions.ExecutionContext;
-import org.apache.flink.runtime.execution.Environment;
-
-
-/**
- * Default implementation of the {@link ExecutionContext} that delegates the calls to the nephele task
- * environment.
- *
- */
-public class RuntimeExecutionContext implements ExecutionContext
-{
- private final Environment env;
-
- public RuntimeExecutionContext(Environment env) {
- this.env = env;
- }
-
-
- @Override
- public String getTaskName() {
- return this.env.getTaskName();
- }
-
-
- @Override
- public int getNumberOfSubtasks() {
- return this.env.getCurrentNumberOfSubtasks();
- }
-
-
- @Override
- public int getSubtaskIndex() {
- return this.env.getIndexInSubtaskGroup() + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bc89e911/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
index aaad08c..99a59b1 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.test.operators;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.record.functions.CrossFunction;
import org.apache.flink.api.java.record.io.DelimitedInputFormat;
@@ -35,8 +32,6 @@ import org.apache.flink.test.util.RecordAPITestBase;
import org.apache.flink.types.IntValue;
import org.apache.flink.types.Record;
import org.apache.flink.types.StringValue;
-import org.apache.flink.util.Collector;
-import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -47,17 +42,12 @@ import java.io.Serializable;
import java.util.Collection;
import java.util.LinkedList;
-/**
- */
@RunWith(Parameterized.class)
-//@Ignore("Test needs to be adapted to new cross signature")
public class CrossITCase extends RecordAPITestBase {
- private static final Log LOG = LogFactory.getLog(CrossITCase.class);
-
- String leftInPath = null;
- String rightInPath = null;
- String resultPath = null;
+ private String leftInPath = null;
+ private String rightInPath = null;
+ private String resultPath = null;
public CrossITCase(Configuration testConfig) {
super(testConfig);
@@ -112,8 +102,6 @@ public class CrossITCase extends RecordAPITestBase {
int key1 = Integer.parseInt(string.toString());
string = record2.getField(0, string);
int key2 = Integer.parseInt(string.toString());
-
- LOG.debug("Processing { [" + key1 + "," + val1 + "] , [" + key2 + "," + val2 + "] }");
string.setValue((key1 + key2 + 2) + "");
integer.setValue(val2 - val1 + 1);
[31/39] git commit: [doku] fix broken image urls
Posted by rm...@apache.org.
[doku] fix broken image urls
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/e4c37542
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/e4c37542
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/e4c37542
Branch: refs/heads/travis_test
Commit: e4c375429c10502c808f2d8e4bf8d443298923dd
Parents: 72d7b86
Author: Robert Metzger <rm...@apache.org>
Authored: Sun Aug 3 14:41:24 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Sun Aug 3 14:41:24 2014 +0200
----------------------------------------------------------------------
docs/internal_general_arch.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/e4c37542/docs/internal_general_arch.md
----------------------------------------------------------------------
diff --git a/docs/internal_general_arch.md b/docs/internal_general_arch.md
index 70ac7be..ce6e810 100644
--- a/docs/internal_general_arch.md
+++ b/docs/internal_general_arch.md
@@ -14,7 +14,7 @@ into the parallel data flow form that is executed by the JobManager and TaskMana
illustrates the different actors in the system very coarsely.
<div style="text-align: center;">
-<img src="ClientJmTm.svg" alt="The Interactions between Client, JobManager and TaskManager" height="400px" style="text-align: center;"/>
+<img src="img/ClientJmTm.svg" alt="The Interactions between Client, JobManager and TaskManager" height="400px" style="text-align: center;"/>
</div>
## Component Stack
@@ -34,7 +34,7 @@ to provide utilities (serializers, comparators) that describe the interaction be
the runtime.
<div style="text-align: center;">
-<img src="stack.svg" alt="The Flink component stack" width="800px" />
+<img src="img/stack.svg" alt="The Flink component stack" width="800px" />
</div>
## Projects and Dependencies
@@ -46,7 +46,7 @@ of smaller sub-modules.
The individual projects and their dependencies are shown in the figure below.
<div style="text-align: center;">
-<img src="projects_dependencies.svg" alt="The Flink sub-projects and their dependencies" height="600px" style="text-align: center;"/>
+<img src="img/projects_dependencies.svg" alt="The Flink sub-projects and their dependencies" height="600px" style="text-align: center;"/>
</div>
In addition to the projects listed in the figure above, Flink currently contains the following sub-projects:
[37/39] git commit: Remove expression key methods
Posted by rm...@apache.org.
Remove expression key methods
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/b63beba5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/b63beba5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/b63beba5
Branch: refs/heads/travis_test
Commit: b63beba56fc995dba87d6fc4766af2900c0438d5
Parents: 1637cb1
Author: uce <u....@fu-berlin.de>
Authored: Fri Aug 8 16:40:57 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Fri Aug 8 17:16:27 2014 +0200
----------------------------------------------------------------------
.../example/java/wordcount/WordCountPOJO.java | 51 ++++++++++----------
.../java/org/apache/flink/api/java/DataSet.java | 6 +--
.../api/java/operators/CoGroupOperator.java | 24 ++++-----
.../flink/api/java/operators/JoinOperator.java | 24 ++++-----
.../api/java/operator/CoGroupOperatorTest.java | 8 +--
.../flink/api/java/operator/GroupingTest.java | 6 +--
.../api/java/operator/JoinOperatorTest.java | 8 +--
7 files changed, 63 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
index d993e60..72d2699 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
@@ -19,7 +19,6 @@
package org.apache.flink.example.java.wordcount;
import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -62,31 +61,31 @@ public class WordCountPOJO {
//
// [1] https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E
// ====================================================================
-
- parseParameters(args);
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // get input data
- DataSet<String> text = getTextDataSet(env);
-
- DataSet<WC> counts = text
- .flatMap(new Tokenizer())
- .groupBy("word")
- .reduce(new ReduceFunction<WC>() {
- public WC reduce(WC value1, WC value2) {
- return new WC(value1.word, value1.count + value2.count);
- }
- });
-
- // emit result
- if(fileOutput) {
- counts.writeAsText(outputPath);
- } else {
- counts.print();
- }
-
- env.execute("WordCount with custom data types example");
+//
+// parseParameters(args);
+//
+// final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+//
+// // get input data
+// DataSet<String> text = getTextDataSet(env);
+//
+// DataSet<WC> counts = text
+// .flatMap(new Tokenizer())
+// .groupBy("word")
+// .reduce(new ReduceFunction<WC>() {
+// public WC reduce(WC value1, WC value2) {
+// return new WC(value1.word, value1.count + value2.count);
+// }
+// });
+//
+// // emit result
+// if(fileOutput) {
+// counts.writeAsText(outputPath);
+// } else {
+// counts.print();
+// }
+//
+// env.execute("WordCount with custom data types example");
}
// *************************************************************************
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index 3d1238a..5da572d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -434,9 +434,9 @@ public abstract class DataSet<T> {
* @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
- public UnsortedGrouping<T> groupBy(String... fields) {
- return new UnsortedGrouping<T>(this, new Keys.ExpressionKeys<T>(fields, getType()));
- }
+// public UnsortedGrouping<T> groupBy(String... fields) {
+// return new UnsortedGrouping<T>(this, new Keys.ExpressionKeys<T>(fields, getType()));
+// }
// --------------------------------------------------------------------------------------------
// Joining
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
index a890b32..c1ec8c6 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
@@ -353,12 +353,12 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @see Tuple
* @see DataSet
*/
- public CoGroupOperatorSetsPredicate where(String field0, String... fields) {
- String[] actualFields = new String[fields.length + 1];
- actualFields[0] = field0;
- System.arraycopy(fields, 0, actualFields, 1, fields.length);
- return new CoGroupOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(actualFields, input1.getType()));
- }
+// public CoGroupOperatorSetsPredicate where(String field0, String... fields) {
+// String[] actualFields = new String[fields.length + 1];
+// actualFields[0] = field0;
+// System.arraycopy(fields, 0, actualFields, 1, fields.length);
+// return new CoGroupOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(actualFields, input1.getType()));
+// }
/**
* Continues a CoGroup transformation and defines a {@link KeySelector} function for the first co-grouped {@link DataSet}.</br>
@@ -426,12 +426,12 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @return An incomplete CoGroup transformation.
* Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.java.functions.RichCoGroupFunction)} to finalize the CoGroup transformation.
*/
- public CoGroupOperatorWithoutFunction equalTo(String field0, String... fields) {
- String[] actualFields = new String[fields.length + 1];
- actualFields[0] = field0;
- System.arraycopy(fields, 0, actualFields, 1, fields.length);
- return createCoGroupOperator(new Keys.ExpressionKeys<I2>(actualFields, input2.getType()));
- }
+// public CoGroupOperatorWithoutFunction equalTo(String field0, String... fields) {
+// String[] actualFields = new String[fields.length + 1];
+// actualFields[0] = field0;
+// System.arraycopy(fields, 0, actualFields, 1, fields.length);
+// return createCoGroupOperator(new Keys.ExpressionKeys<I2>(actualFields, input2.getType()));
+// }
/**
* Continues a CoGroup transformation and defines a {@link KeySelector} function for the second co-grouped {@link DataSet}.</br>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
index ce0aea7..f446193 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
@@ -743,12 +743,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @see Tuple
* @see DataSet
*/
- public JoinOperatorSetsPredicate where(String field0, String... fields) {
- String[] actualFields = new String[fields.length + 1];
- actualFields[0] = field0;
- System.arraycopy(fields, 0, actualFields, 1, fields.length);
- return new JoinOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(actualFields, input1.getType()));
- }
+// public JoinOperatorSetsPredicate where(String field0, String... fields) {
+// String[] actualFields = new String[fields.length + 1];
+// actualFields[0] = field0;
+// System.arraycopy(fields, 0, actualFields, 1, fields.length);
+// return new JoinOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(actualFields, input1.getType()));
+// }
/**
* Continues a Join transformation and defines a {@link KeySelector} function for the first join {@link DataSet}.</br>
@@ -825,12 +825,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param fields The fields of the second join DataSet that should be used as keys.
* @return A DefaultJoin that represents the joined DataSet.
*/
- public DefaultJoin<I1, I2> equalTo(String field0, String... fields) {
- String[] actualFields = new String[fields.length + 1];
- actualFields[0] = field0;
- System.arraycopy(fields, 0, actualFields, 1, fields.length);
- return createJoinOperator(new Keys.ExpressionKeys<I2>(actualFields, input2.getType()));
- }
+// public DefaultJoin<I1, I2> equalTo(String field0, String... fields) {
+// String[] actualFields = new String[fields.length + 1];
+// actualFields[0] = field0;
+// System.arraycopy(fields, 0, actualFields, 1, fields.length);
+// return createJoinOperator(new Keys.ExpressionKeys<I2>(actualFields, input2.getType()));
+// }
/**
* Continues a Join transformation and defines a {@link KeySelector} function for the second join {@link DataSet}.</br>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
index 360050b..20fea28 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
@@ -139,7 +139,7 @@ public class CoGroupOperatorTest {
// should work
try {
- ds1.coGroup(ds2).where("myInt").equalTo("myInt");
+// ds1.coGroup(ds2).where("myInt").equalTo("myInt");
} catch(Exception e) {
Assert.fail();
}
@@ -154,7 +154,7 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible cogroup key types
- ds1.coGroup(ds2).where("myInt").equalTo("myString");
+// ds1.coGroup(ds2).where("myInt").equalTo("myString");
}
@Ignore
@@ -166,7 +166,7 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible number of cogroup keys
- ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
+// ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
}
@Ignore
@@ -178,7 +178,7 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, cogroup key non-existent
- ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
+// ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
index 98b6998..f5cd7b9 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
@@ -125,7 +125,7 @@ public class GroupingTest {
// should work
try {
- ds.groupBy("myInt");
+// ds.groupBy("myInt");
} catch(Exception e) {
Assert.fail();
}
@@ -139,7 +139,7 @@ public class GroupingTest {
DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
// should not work: groups on basic type
- longDs.groupBy("myInt");
+// longDs.groupBy("myInt");
}
@Ignore
@@ -164,7 +164,7 @@ public class GroupingTest {
DataSet<CustomType> ds = env.fromCollection(customTypeData);
// should not work, key out of tuple bounds
- ds.groupBy("myNonExistent");
+// ds.groupBy("myNonExistent");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b63beba5/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
index 11eca83..fe4414b 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
@@ -139,7 +139,7 @@ public class JoinOperatorTest {
// should work
try {
- ds1.join(ds2).where("myInt").equalTo("myInt");
+// ds1.join(ds2).where("myInt").equalTo("myInt");
} catch(Exception e) {
Assert.fail();
}
@@ -154,7 +154,7 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible join key types
- ds1.join(ds2).where("myInt").equalTo("myString");
+// ds1.join(ds2).where("myInt").equalTo("myString");
}
@Ignore
@@ -166,7 +166,7 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible number of join keys
- ds1.join(ds2).where("myInt", "myString").equalTo("myString");
+// ds1.join(ds2).where("myInt", "myString").equalTo("myString");
}
@Ignore
@@ -178,7 +178,7 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, join key non-existent
- ds1.join(ds2).where("myNonExistent").equalTo("myInt");
+// ds1.join(ds2).where("myNonExistent").equalTo("myInt");
}
[13/39] git commit: [Docs] Limit nesting of generated TOCs
Posted by rm...@apache.org.
[Docs] Limit nesting of generated TOCs
The jekyll tocify plugin generates an HTML TOC based on the headings
of the markdown files. This commit limits the nesting of the TOC to 2,
which means that only h1 and h2 element will be taken into account.
Previously, all heading elements were used, which resulted in a very
cluttered TOC.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/bcf85c2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/bcf85c2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/bcf85c2a
Branch: refs/heads/travis_test
Commit: bcf85c2a50326dd3e54df223a2494dcc4f83eab5
Parents: 23289d6
Author: uce <u....@fu-berlin.de>
Authored: Thu Jul 31 15:22:10 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Thu Jul 31 15:22:10 2014 +0200
----------------------------------------------------------------------
docs/_plugins/tocify.rb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bcf85c2a/docs/_plugins/tocify.rb
----------------------------------------------------------------------
diff --git a/docs/_plugins/tocify.rb b/docs/_plugins/tocify.rb
index 7df0c3d..9eb6293 100644
--- a/docs/_plugins/tocify.rb
+++ b/docs/_plugins/tocify.rb
@@ -1,7 +1,7 @@
module Jekyll
module Tocify
def tocify(input)
- converter = Redcarpet::Markdown.new(Redcarpet::Render::HTML_TOC)
+ converter = Redcarpet::Markdown.new(Redcarpet::Render::HTML_TOC.new(nesting_level: 2))
converter.render(input)
end
end
[23/39] git commit: [FLINK-701] Refactor Java API to use SAM
interfaces. Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
[FLINK-701] Refactor Java API to use SAM interfaces. Introduce RichFunction stubs for all UDFs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/22b24f20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/22b24f20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/22b24f20
Branch: refs/heads/travis_test
Commit: 22b24f208de2501311fe504e767526690742e30e
Parents: 9d1c49f
Author: Kostas Tzoumas <Ko...@gmail.com>
Authored: Fri Jul 18 15:37:19 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 31 17:47:35 2014 +0200
----------------------------------------------------------------------
.../flink/api/avro/AvroOutputFormatTest.java | 6 +-
.../avro/testjar/AvroExternalJarProgram.java | 8 +-
.../mapred/example/WordCount.java | 8 +-
.../mapreduce/example/WordCount.java | 8 +-
.../spargel/java/VertexCentricIteration.java | 8 +-
.../examples/SpargelConnectedComponents.java | 4 +-
.../spargel/java/examples/SpargelPageRank.java | 8 +-
.../SpargelPageRankCountingVertices.java | 14 +-
.../SpargelConnectedComponentsITCase.java | 4 +-
.../apache/flink/client/testjar/WordCount.java | 4 +-
.../flink/compiler/util/NoOpFunction.java | 4 +-
.../compiler/BranchingPlansCompilerTest.java | 5 +-
.../CachedMatchStrategyCompilerTest.java | 5 +-
.../compiler/CoGroupSolutionSetFirstTest.java | 9 +-
.../apache/flink/compiler/CompilerTestBase.java | 8 +-
.../compiler/FeedbackPropertiesMatchTest.java | 10 +-
.../compiler/GroupReduceCompilationTest.java | 20 +-
.../flink/compiler/IterationsCompilerTest.java | 20 +-
.../flink/compiler/ReduceCompilationTest.java | 10 +-
.../compiler/UnionPropertyPropagationTest.java | 4 +-
.../WorksetIterationsJavaApiCompilerTest.java | 18 +-
.../testfunctions/DummyFlatJoinFunction.java | 33 ++
.../testfunctions/DummyJoinFunction.java | 32 -
.../testfunctions/IdentityGroupReducer.java | 6 +-
.../compiler/testfunctions/IdentityMapper.java | 4 +-
.../testfunctions/SelectOneReducer.java | 4 +-
.../testfunctions/Top1GroupReducer.java | 6 +-
.../flink/compiler/util/DummyCrossStub.java | 9 +-
.../api/common/functions/AbstractFunction.java | 76 ---
.../common/functions/AbstractRichFunction.java | 76 +++
.../api/common/functions/CoGroupFunction.java | 41 ++
.../api/common/functions/CombineFunction.java | 31 +
.../api/common/functions/CrossFunction.java | 42 ++
.../api/common/functions/FilterFunction.java | 33 ++
.../common/functions/FlatCombineFunction.java | 33 ++
.../api/common/functions/FlatJoinFunction.java | 30 +
.../api/common/functions/FlatMapFunction.java | 45 ++
.../flink/api/common/functions/Function.java | 75 +--
.../api/common/functions/GenericCoGrouper.java | 40 --
.../common/functions/GenericCollectorMap.java | 2 +-
.../api/common/functions/GenericCombine.java | 32 -
.../api/common/functions/GenericCrosser.java | 41 --
.../api/common/functions/GenericFilter.java | 33 --
.../api/common/functions/GenericFlatMap.java | 42 --
.../common/functions/GenericGroupReduce.java | 44 --
.../api/common/functions/GenericJoiner.java | 28 -
.../flink/api/common/functions/GenericMap.java | 30 -
.../api/common/functions/GenericReduce.java | 26 -
.../common/functions/GroupReduceFunction.java | 45 ++
.../api/common/functions/JoinFunction.java | 28 +
.../flink/api/common/functions/MapFunction.java | 38 ++
.../api/common/functions/ReduceFunction.java | 39 ++
.../api/common/functions/RichFunction.java | 97 ++++
.../api/common/functions/RuntimeContext.java | 2 +-
.../common/functions/util/FunctionUtils.java | 81 +++
.../flink/api/common/operators/Union.java | 6 +-
.../operators/base/BulkIterationBase.java | 8 +-
.../operators/base/CoGroupOperatorBase.java | 6 +-
.../operators/base/CrossOperatorBase.java | 6 +-
.../operators/base/DeltaIterationBase.java | 6 +-
.../operators/base/FilterOperatorBase.java | 6 +-
.../operators/base/FlatMapOperatorBase.java | 6 +-
.../operators/base/GroupReduceOperatorBase.java | 17 +-
.../common/operators/base/JoinOperatorBase.java | 6 +-
.../common/operators/base/MapOperatorBase.java | 4 +-
.../operators/base/ReduceOperatorBase.java | 6 +-
.../api/common/operators/util/OperatorUtil.java | 18 +-
.../common/operators/util/OperatorUtilTest.java | 21 +-
flink-examples/flink-java-examples/pom.xml | 1 -
.../flink/example/java/clustering/KMeans.java | 17 +-
.../example/java/graph/ConnectedComponents.java | 24 +-
.../example/java/graph/EnumTrianglesBasic.java | 14 +-
.../example/java/graph/EnumTrianglesOpt.java | 28 +-
.../flink/example/java/graph/PageRankBasic.java | 18 +-
.../java/graph/TransitiveClosureNaive.java | 4 +-
.../flink/example/java/ml/LinearRegression.java | 15 +-
.../relational/EmptyFieldsCountAccumulator.java | 4 +-
.../java/relational/RelationalQuery.java | 2 +-
.../example/java/relational/TPCHQuery10.java | 4 +-
.../example/java/relational/TPCHQuery3.java | 4 +-
.../example/java/relational/WebLogAnalysis.java | 12 +-
.../flink/example/java/wordcount/WordCount.java | 4 +-
.../example/java/wordcount/WordCountPOJO.java | 6 +-
flink-examples/pom.xml | 1 -
.../java/org/apache/flink/api/java/DataSet.java | 93 +--
.../apache/flink/api/java/DeltaIteration.java | 2 +-
.../flink/api/java/ExecutionEnvironment.java | 6 +-
.../apache/flink/api/java/IterativeDataSet.java | 2 +-
.../api/java/functions/CoGroupFunction.java | 74 ---
.../flink/api/java/functions/CrossFunction.java | 76 ---
.../api/java/functions/FilterFunction.java | 57 --
.../api/java/functions/FlatMapFunction.java | 59 --
.../api/java/functions/FlatMapIterator.java | 4 +-
.../api/java/functions/FunctionAnnotation.java | 16 +-
.../api/java/functions/GroupReduceFunction.java | 114 ----
.../api/java/functions/GroupReduceIterator.java | 2 +-
.../flink/api/java/functions/JoinFunction.java | 93 ---
.../flink/api/java/functions/MapFunction.java | 59 --
.../api/java/functions/ReduceFunction.java | 63 --
.../api/java/functions/RichCoGroupFunction.java | 74 +++
.../api/java/functions/RichCrossFunction.java | 67 +++
.../api/java/functions/RichFilterFunction.java | 57 ++
.../java/functions/RichFlatCombineFunction.java | 34 ++
.../java/functions/RichFlatJoinFunction.java | 75 +++
.../api/java/functions/RichFlatMapFunction.java | 59 ++
.../java/functions/RichGroupReduceFunction.java | 114 ++++
.../api/java/functions/RichJoinFunction.java | 31 +
.../api/java/functions/RichMapFunction.java | 59 ++
.../api/java/functions/RichReduceFunction.java | 63 ++
.../UnsupportedLambdaExpressionException.java | 30 +
.../api/java/operators/AggregateOperator.java | 20 +-
.../api/java/operators/CoGroupOperator.java | 52 +-
.../flink/api/java/operators/CrossOperator.java | 18 +-
.../api/java/operators/DistinctOperator.java | 44 +-
.../api/java/operators/FilterOperator.java | 6 +-
.../api/java/operators/FlatMapOperator.java | 7 +-
.../api/java/operators/GroupReduceOperator.java | 224 +++++++
.../flink/api/java/operators/Grouping.java | 4 +-
.../flink/api/java/operators/JoinOperator.java | 163 ++++--
.../flink/api/java/operators/MapOperator.java | 10 +-
.../api/java/operators/ProjectOperator.java | 4 +-
.../api/java/operators/ReduceGroupOperator.java | 215 -------
.../api/java/operators/ReduceOperator.java | 19 +-
.../java/operators/SingleInputUdfOperator.java | 4 +-
.../api/java/operators/SortedGrouping.java | 20 +-
.../api/java/operators/TwoInputUdfOperator.java | 4 +-
.../flink/api/java/operators/UdfOperator.java | 10 +-
.../api/java/operators/UnsortedGrouping.java | 24 +-
.../translation/KeyExtractingMapper.java | 4 +-
.../translation/KeyRemovingMapper.java | 4 +-
.../translation/PlanFilterOperator.java | 8 +-
.../translation/PlanProjectOperator.java | 10 +-
.../PlanUnwrappingCoGroupOperator.java | 13 +-
.../translation/PlanUnwrappingJoinOperator.java | 28 +-
.../PlanUnwrappingReduceGroupOperator.java | 21 +-
.../PlanUnwrappingReduceOperator.java | 7 +-
.../translation/TupleKeyExtractingMapper.java | 4 +-
.../operators/translation/WrappingFunction.java | 22 +-
.../java/record/functions/CoGroupFunction.java | 5 +-
.../java/record/functions/CrossFunction.java | 19 +-
.../api/java/record/functions/JoinFunction.java | 8 +-
.../api/java/record/functions/MapFunction.java | 4 +-
.../java/record/functions/ReduceFunction.java | 8 +-
.../flink/api/java/typeutils/TypeExtractor.java | 66 ++-
.../java/typeutils/runtime/TupleComparator.java | 2 +-
.../SemanticPropertiesTranslationTest.java | 12 +-
.../DeltaIterationTranslationTest.java | 26 +-
.../translation/ReduceTranslationTests.java | 10 +-
.../java/type/extractor/TypeExtractorTest.java | 126 ++--
flink-java8-tests/pom.xml | 145 +++++
.../javaApiOperators/lambdas/CoGroupITCase.java | 70 +++
.../javaApiOperators/lambdas/CrossITCase.java | 59 ++
.../javaApiOperators/lambdas/FilterITCase.java | 142 +++++
.../lambdas/FlatJoinITCase.java | 58 ++
.../javaApiOperators/lambdas/FlatMapITCase.java | 46 ++
.../lambdas/GroupReduceITCase.java | 84 +++
.../javaApiOperators/lambdas/JoinITCase.java | 58 ++
.../javaApiOperators/lambdas/MapITCase.java | 48 ++
.../javaApiOperators/lambdas/ReduceITCase.java | 160 +++++
.../task/AbstractIterativePactTask.java | 4 +-
.../iterative/task/IterationHeadPactTask.java | 4 +-
.../task/IterationIntermediatePactTask.java | 4 +-
.../iterative/task/IterationTailPactTask.java | 4 +-
.../AbstractCachedBuildSideMatchDriver.java | 6 +-
.../runtime/operators/AllGroupReduceDriver.java | 24 +-
.../runtime/operators/AllReduceDriver.java | 16 +-
.../flink/runtime/operators/CoGroupDriver.java | 14 +-
.../CoGroupWithSolutionSetFirstDriver.java | 14 +-
.../CoGroupWithSolutionSetSecondDriver.java | 14 +-
.../flink/runtime/operators/CrossDriver.java | 32 +-
.../flink/runtime/operators/FlatMapDriver.java | 16 +-
.../operators/GroupReduceCombineDriver.java | 14 +-
.../runtime/operators/GroupReduceDriver.java | 16 +-
.../JoinWithSolutionSetFirstDriver.java | 14 +-
.../JoinWithSolutionSetSecondDriver.java | 14 +-
.../flink/runtime/operators/MapDriver.java | 16 +-
.../flink/runtime/operators/MatchDriver.java | 16 +-
.../flink/runtime/operators/NoOpDriver.java | 10 +-
.../runtime/operators/ReduceCombineDriver.java | 16 +-
.../flink/runtime/operators/ReduceDriver.java | 16 +-
.../runtime/operators/RegularPactTask.java | 42 +-
.../chaining/ChainedCollectorMapDriver.java | 4 +-
.../chaining/ChainedFlatMapDriver.java | 17 +-
.../operators/chaining/ChainedMapDriver.java | 14 +-
.../ChainedTerminationCriterionDriver.java | 4 +-
.../SynchronousChainedCombineDriver.java | 13 +-
.../hash/BuildFirstHashMatchIterator.java | 4 +-
.../hash/BuildSecondHashMatchIterator.java | 4 +-
.../sort/CombiningUnilateralSortMerger.java | 19 +-
.../operators/sort/MergeMatchIterator.java | 14 +-
.../operators/util/JoinTaskIterator.java | 4 +-
.../runtime/operators/CachedMatchTaskTest.java | 7 +-
.../operators/CoGroupTaskExternalITCase.java | 9 +-
.../runtime/operators/CoGroupTaskTest.java | 11 +-
.../operators/CombineTaskExternalITCase.java | 6 +-
.../runtime/operators/CombineTaskTest.java | 6 +-
.../operators/CrossTaskExternalITCase.java | 6 +-
.../flink/runtime/operators/CrossTaskTest.java | 22 +-
.../operators/MatchTaskExternalITCase.java | 6 +-
.../flink/runtime/operators/MatchTaskTest.java | 14 +-
.../operators/ReduceTaskExternalITCase.java | 6 +-
.../flink/runtime/operators/ReduceTaskTest.java | 6 +-
.../drivers/AllGroupReduceDriverTest.java | 20 +-
.../operators/drivers/AllReduceDriverTest.java | 32 +-
.../drivers/GroupReduceDriverTest.java | 20 +-
.../drivers/ReduceCombineDriverTest.java | 32 +-
.../operators/drivers/ReduceDriverTest.java | 32 +-
.../operators/hash/HashMatchIteratorITCase.java | 18 +-
.../sort/SortMergeMatchIteratorITCase.java | 4 +-
.../operators/testutils/DriverTestBase.java | 8 +-
.../operators/testutils/TaskTestBase.java | 6 +-
.../operators/util/HashVsSortMiniBenchmark.java | 4 +-
.../api/scala/functions/CrossFunction.scala | 28 +-
.../api/scala/operators/CrossOperator.scala | 69 +--
.../api/scala/operators/IterateOperators.scala | 2 +-
.../test/compiler/util/CompilerTestBase.java | 4 +-
.../test/compiler/util/OperatorResolver.java | 4 +-
flink-tests/pom.xml | 2 +-
...ultipleJoinsWithSolutionSetCompilerTest.java | 14 +-
.../BulkIterationWithAllReducerITCase.java | 4 +-
.../CoGroupConnectedComponentsSecondITCase.java | 12 +-
.../DependencyConnectedComponentsITCase.java | 23 +-
.../aggregators/AggregatorsITCase.java | 20 +-
...nentsWithParametrizableAggregatorITCase.java | 15 +-
...entsWithParametrizableConvergenceITCase.java | 15 +-
.../CustomCompensatableDotProductCoGroup.java | 6 +-
.../CustomCompensatableDotProductMatch.java | 8 +-
.../CustomCompensatingMap.java | 4 +-
.../CustomRankCombiner.java | 10 +-
.../test/javaApiOperators/CoGroupITCase.java | 17 +-
.../test/javaApiOperators/CrossITCase.java | 15 +-
.../test/javaApiOperators/DistinctITCase.java | 4 +-
.../test/javaApiOperators/FilterITCase.java | 9 +-
.../test/javaApiOperators/FlatMapITCase.java | 5 +-
.../javaApiOperators/GroupReduceITCase.java | 578 ++++++++++---------
.../flink/test/javaApiOperators/JoinITCase.java | 105 ++--
.../flink/test/javaApiOperators/MapITCase.java | 7 +-
.../test/javaApiOperators/ReduceITCase.java | 19 +-
.../test/javaApiOperators/UnionITCase.java | 4 +-
.../flink/test/operators/CrossITCase.java | 49 +-
.../recordJobs/kmeans/udfs/ComputeDistance.java | 16 +-
.../test/recordJobs/relational/TPCHQuery3.java | 2 +-
.../flink/test/util/testjar/KMeansForTest.java | 16 +-
pom.xml | 3 +
244 files changed, 4106 insertions(+), 2872 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
index 637a5e9..386f318 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroOutputFormatTest.java
@@ -30,7 +30,7 @@ import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.io.AvroOutputFormat;
import org.apache.flink.api.java.record.io.avro.example.User;
import org.apache.flink.api.java.tuple.Tuple3;
@@ -125,7 +125,7 @@ public class AvroOutputFormatTest extends JavaProgramTestBase {
}
- public final static class ConvertToUser extends MapFunction<Tuple3<String, Integer, String>, User> {
+ public final static class ConvertToUser extends RichMapFunction<Tuple3<String, Integer, String>, User> {
@Override
public User map(Tuple3<String, Integer, String> value) throws Exception {
@@ -133,7 +133,7 @@ public class AvroOutputFormatTest extends JavaProgramTestBase {
}
}
- public final static class ConvertToReflective extends MapFunction<User, ReflectiveUser> {
+ public final static class ConvertToReflective extends RichMapFunction<User, ReflectiveUser> {
@Override
public ReflectiveUser map(User value) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
index 146c72b..75b7da6 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/testjar/AvroExternalJarProgram.java
@@ -40,8 +40,8 @@ import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.flink.api.avro.AvroBaseValue;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -141,7 +141,7 @@ public class AvroExternalJarProgram {
// --------------------------------------------------------------------------------------------
- public static final class NameExtractor extends MapFunction<MyUser, Tuple2<String, MyUser>> {
+ public static final class NameExtractor extends RichMapFunction<MyUser, Tuple2<String, MyUser>> {
private static final long serialVersionUID = 1L;
@Override
@@ -151,7 +151,7 @@ public class AvroExternalJarProgram {
}
}
- public static final class NameGrouper extends ReduceFunction<Tuple2<String, MyUser>> {
+ public static final class NameGrouper extends RichReduceFunction<Tuple2<String, MyUser>> {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/WordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/WordCount.java
index 4e8ffa9..ba09e77 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/WordCount.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/example/WordCount.java
@@ -18,6 +18,7 @@
package org.apache.flink.hadoopcompatibility.mapred.example;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
@@ -28,8 +29,7 @@ import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.hadoopcompatibility.mapred.HadoopInputFormat;
import org.apache.flink.hadoopcompatibility.mapred.HadoopOutputFormat;
@@ -88,7 +88,7 @@ public class WordCount {
/**
* Splits a line into words and converts Hadoop Writables into normal Java data types.
*/
- public static final class Tokenizer extends FlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
+ public static final class Tokenizer extends RichFlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
@Override
public void flatMap(Tuple2<LongWritable, Text> value, Collector<Tuple2<String, Integer>> out) {
@@ -108,7 +108,7 @@ public class WordCount {
/**
* Converts Java data types to Hadoop Writables.
*/
- public static final class HadoopDatatypeMapper extends MapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
+ public static final class HadoopDatatypeMapper extends RichMapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
@Override
public Tuple2<Text, IntWritable> map(Tuple2<String, Integer> value) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
index 36ea378..c00a14a 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapreduce/example/WordCount.java
@@ -19,8 +19,8 @@
package org.apache.flink.hadoopcompatibility.mapreduce.example;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
import org.apache.hadoop.fs.Path;
@@ -89,7 +89,7 @@ public class WordCount {
/**
* Splits a line into words and converts Hadoop Writables into normal Java data types.
*/
- public static final class Tokenizer extends FlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
+ public static final class Tokenizer extends RichFlatMapFunction<Tuple2<LongWritable, Text>, Tuple2<String, Integer>> {
@Override
public void flatMap(Tuple2<LongWritable, Text> value, Collector<Tuple2<String, Integer>> out) {
@@ -109,7 +109,7 @@ public class WordCount {
/**
* Converts Java data types to Hadoop Writables.
*/
- public static final class HadoopDatatypeMapper extends MapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
+ public static final class HadoopDatatypeMapper extends RichMapFunction<Tuple2<String, Integer>, Tuple2<Text, IntWritable>> {
@Override
public Tuple2<Text, IntWritable> map(Tuple2<String, Integer> value) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
index bb84cea..65be2f8 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
@@ -29,7 +29,7 @@ import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
-import org.apache.flink.api.java.functions.CoGroupFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
import org.apache.flink.api.java.operators.CoGroupOperator;
import org.apache.flink.api.java.operators.CustomUnaryOperation;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -393,7 +393,7 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
// --------------------------------------------------------------------------------------------
private static final class VertexUpdateUdf<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
- extends CoGroupFunction<Tuple2<VertexKey, Message>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
+ extends RichCoGroupFunction<Tuple2<VertexKey, Message>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, VertexValue>>
implements ResultTypeQueryable<Tuple2<VertexKey, VertexValue>>
{
private static final long serialVersionUID = 1L;
@@ -463,7 +463,7 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
* UDF that encapsulates the message sending function for graphs where the edges have no associated values.
*/
private static final class MessagingUdfNoEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message>
- extends CoGroupFunction<Tuple2<VertexKey, VertexKey>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
+ extends RichCoGroupFunction<Tuple2<VertexKey, VertexKey>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
{
private static final long serialVersionUID = 1L;
@@ -516,7 +516,7 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
* UDF that encapsulates the message sending function for graphs where the edges have an associated value.
*/
private static final class MessagingUdfWithEdgeValues<VertexKey extends Comparable<VertexKey>, VertexValue, Message, EdgeValue>
- extends CoGroupFunction<Tuple3<VertexKey, VertexKey, EdgeValue>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
+ extends RichCoGroupFunction<Tuple3<VertexKey, VertexKey, EdgeValue>, Tuple2<VertexKey, VertexValue>, Tuple2<VertexKey, Message>>
implements ResultTypeQueryable<Tuple2<VertexKey, Message>>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
index ea90feb..a4ba6fa 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelConnectedComponents.java
@@ -18,7 +18,7 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.spargel.java.MessageIterator;
import org.apache.flink.spargel.java.MessagingFunction;
@@ -70,7 +70,7 @@ public class SpargelConnectedComponents {
* A map function that takes a Long value and creates a 2-tuple out of it:
* <pre>(Long value) -> (value, value)</pre>
*/
- public static final class IdAssigner extends MapFunction<Long, Tuple2<Long, Long>> {
+ public static final class IdAssigner extends RichMapFunction<Long, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> map(Long value) {
return new Tuple2<Long, Long>(value, value);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
index c7fbaaa..9dfc327 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRank.java
@@ -18,8 +18,8 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.spargel.java.MessageIterator;
@@ -48,7 +48,7 @@ public class SpargelPageRank {
// enumerate some sample edges and assign an initial uniform probability (rank)
DataSet<Tuple2<Long, Double>> intialRanks = env.generateSequence(1, numVertices)
- .map(new MapFunction<Long, Tuple2<Long, Double>>() {
+ .map(new RichMapFunction<Long, Tuple2<Long, Double>>() {
public Tuple2<Long, Double> map(Long value) {
return new Tuple2<Long, Double>(value, 1.0/numVertices);
}
@@ -56,7 +56,7 @@ public class SpargelPageRank {
// generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, numVertices)
- .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ .flatMap(new RichFlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
int numOutEdges = (int) (Math.random() * (numVertices / 2));
for (int i = 0; i < numOutEdges; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
index 34c9ad8..43c0b84 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/examples/SpargelPageRankCountingVertices.java
@@ -18,9 +18,9 @@
package org.apache.flink.spargel.java.examples;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
@@ -53,7 +53,7 @@ public class SpargelPageRankCountingVertices {
// generate some random edges. the transition probability on each edge is 1/num-out-edges of the source vertex
DataSet<Tuple3<Long, Long, Double>> edgesWithProbability = env.generateSequence(1, NUM_VERTICES)
- .flatMap(new FlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
+ .flatMap(new RichFlatMapFunction<Long, Tuple3<Long, Long, Double>>() {
public void flatMap(Long value, Collector<Tuple3<Long, Long, Double>> out) {
int numOutEdges = (int) (Math.random() * (NUM_VERTICES / 2));
for (int i = 0; i < numOutEdges; i++) {
@@ -67,12 +67,12 @@ public class SpargelPageRankCountingVertices {
// count the number of vertices
DataSet<Long> count = vertices
- .map(new MapFunction<Long, Long>() {
+ .map(new RichMapFunction<Long, Long>() {
public Long map(Long value) {
return 1L;
}
})
- .reduce(new ReduceFunction<Long>() {
+ .reduce(new RichReduceFunction<Long>() {
public Long reduce(Long value1, Long value2) {
return value1 + value2;
}
@@ -80,7 +80,7 @@ public class SpargelPageRankCountingVertices {
// enumerate some sample edges and assign an initial uniform probability (rank)
DataSet<Tuple2<Long, Double>> intialRanks = vertices
- .map(new MapFunction<Long, Tuple2<Long, Double>>() {
+ .map(new RichMapFunction<Long, Tuple2<Long, Double>>() {
private long numVertices;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java b/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
index a34f2db..16b004c 100644
--- a/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
+++ b/flink-addons/flink-spargel/src/test/java/org/apache/flink/test/spargel/SpargelConnectedComponentsITCase.java
@@ -21,7 +21,7 @@ package org.apache.flink.test.spargel;
import java.io.BufferedReader;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -72,7 +72,7 @@ public class SpargelConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class EdgeParser extends MapFunction<String, Tuple2<Long, Long>> {
+ public static final class EdgeParser extends RichMapFunction<String, Tuple2<Long, Long>> {
public Tuple2<Long, Long> map(String value) {
String[] nums = value.split(" ");
return new Tuple2<Long, Long>(Long.parseLong(nums[0]), Long.parseLong(nums[1]));
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-clients/src/test/java/org/apache/flink/client/testjar/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/testjar/WordCount.java b/flink-clients/src/test/java/org/apache/flink/client/testjar/WordCount.java
index 7320b7b..2b64b84 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/testjar/WordCount.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/testjar/WordCount.java
@@ -19,7 +19,7 @@
package org.apache.flink.client.testjar;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
@@ -75,7 +75,7 @@ public class WordCount {
* FlatMapFunction. The function takes a line (String) and splits it into
* multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
*/
- public static final class Tokenizer extends FlatMapFunction<String, Tuple2<String, Integer>> {
+ public static final class Tokenizer extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/main/java/org/apache/flink/compiler/util/NoOpFunction.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/compiler/util/NoOpFunction.java b/flink-compiler/src/main/java/org/apache/flink/compiler/util/NoOpFunction.java
index 9f10be5..6eccc8a 100644
--- a/flink-compiler/src/main/java/org/apache/flink/compiler/util/NoOpFunction.java
+++ b/flink-compiler/src/main/java/org/apache/flink/compiler/util/NoOpFunction.java
@@ -18,9 +18,9 @@
package org.apache.flink.compiler.util;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
-public class NoOpFunction extends AbstractFunction {
+public class NoOpFunction extends AbstractRichFunction {
private static final long serialVersionUID = 1L;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/BranchingPlansCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/BranchingPlansCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/BranchingPlansCompilerTest.java
index 571f4e4..10fa34f 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/BranchingPlansCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/BranchingPlansCompilerTest.java
@@ -30,7 +30,7 @@ import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.record.operators.BulkIteration;
import org.apache.flink.api.java.record.operators.CoGroupOperator;
import org.apache.flink.api.java.record.operators.CrossOperator;
@@ -40,7 +40,6 @@ import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.JoinOperator;
import org.apache.flink.api.java.record.operators.MapOperator;
import org.apache.flink.api.java.record.operators.ReduceOperator;
-import org.apache.flink.compiler.PactCompiler;
import org.apache.flink.compiler.plan.OptimizedPlan;
import org.apache.flink.compiler.plan.SinkPlanNode;
import org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator;
@@ -959,7 +958,7 @@ public class BranchingPlansCompilerTest extends CompilerTestBase {
result1.join(result2)
.where(new IdentityKeyExtractor<String>())
.equalTo(new IdentityKeyExtractor<String>())
- .with(new JoinFunction<String, String, String>() {
+ .with(new RichJoinFunction<String, String, String>() {
@Override
public String join(String first, String second) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/CachedMatchStrategyCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/CachedMatchStrategyCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/CachedMatchStrategyCompilerTest.java
index 55c53f0..8226dbf 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/CachedMatchStrategyCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/CachedMatchStrategyCompilerTest.java
@@ -28,9 +28,8 @@ import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.compiler.PactCompiler;
import org.apache.flink.compiler.dag.TempMode;
import org.apache.flink.compiler.plan.DualInputPlanNode;
import org.apache.flink.compiler.plan.OptimizedPlan;
@@ -256,7 +255,7 @@ public class CachedMatchStrategyCompilerTest extends CompilerTestBase {
}
- private static class DummyJoiner extends JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> {
+ private static class DummyJoiner extends RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>> {
@Override
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
index 1c30545..3624d86 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
@@ -21,16 +21,15 @@ package org.apache.flink.compiler;
import java.util.Iterator;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.junit.Assert;
import org.junit.Test;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.compiler.CompilerException;
import org.apache.flink.compiler.plan.Channel;
import org.apache.flink.compiler.plan.DualInputPlanNode;
import org.apache.flink.compiler.plan.OptimizedPlan;
@@ -43,13 +42,13 @@ import org.apache.flink.util.Visitor;
@SuppressWarnings("serial")
public class CoGroupSolutionSetFirstTest extends CompilerTestBase {
- public static class SimpleCGroup extends CoGroupFunction<Tuple1<Integer>, Tuple1<Integer>, Tuple1<Integer>> {
+ public static class SimpleCGroup extends RichCoGroupFunction<Tuple1<Integer>, Tuple1<Integer>, Tuple1<Integer>> {
@Override
public void coGroup(Iterator<Tuple1<Integer>> first, Iterator<Tuple1<Integer>> second, Collector<Tuple1<Integer>> out) throws Exception {
}
}
- public static class SimpleMap extends MapFunction<Tuple1<Integer>, Tuple1<Integer>> {
+ public static class SimpleMap extends RichMapFunction<Tuple1<Integer>, Tuple1<Integer>> {
@Override
public Tuple1<Integer> map(Tuple1<Integer> value) throws Exception {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java b/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
index ff4d6b0..b2c163b 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/CompilerTestBase.java
@@ -27,14 +27,12 @@ import java.util.Map;
import java.util.Set;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
import org.apache.flink.api.java.record.operators.BulkIteration;
import org.apache.flink.api.java.record.operators.DeltaIteration;
-import org.apache.flink.compiler.DataStatistics;
-import org.apache.flink.compiler.PactCompiler;
import org.apache.flink.compiler.costs.DefaultCostEstimator;
import org.apache.flink.compiler.plan.OptimizedPlan;
import org.apache.flink.compiler.plan.PlanNode;
@@ -181,7 +179,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
}
@SuppressWarnings("unchecked")
- public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
+ public <T extends PlanNode> T getNode(String name, Class<? extends RichFunction> stubClass) {
List<PlanNode> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
@@ -243,7 +241,7 @@ public abstract class CompilerTestBase implements java.io.Serializable {
}
@SuppressWarnings("unchecked")
- public <T extends Operator<?>> T getNode(String name, Class<? extends Function> stubClass) {
+ public <T extends Operator<?>> T getNode(String name, Class<? extends RichFunction> stubClass) {
List<Operator<?>> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/FeedbackPropertiesMatchTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/FeedbackPropertiesMatchTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/FeedbackPropertiesMatchTest.java
index d50a7d6..0fbf072 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/FeedbackPropertiesMatchTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/FeedbackPropertiesMatchTest.java
@@ -22,8 +22,8 @@ package org.apache.flink.compiler;
import static org.apache.flink.compiler.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport.*;
import static org.junit.Assert.*;
-import org.apache.flink.api.common.functions.GenericJoiner;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.OperatorInformation;
import org.apache.flink.api.common.operators.Order;
@@ -47,7 +47,7 @@ import org.apache.flink.compiler.plan.DualInputPlanNode;
import org.apache.flink.compiler.plan.SingleInputPlanNode;
import org.apache.flink.compiler.plan.SourcePlanNode;
import org.apache.flink.compiler.plan.PlanNode.FeedbackPropertiesMeetRequirementsReport;
-import org.apache.flink.compiler.testfunctions.DummyJoinFunction;
+import org.apache.flink.compiler.testfunctions.DummyFlatJoinFunction;
import org.apache.flink.compiler.testfunctions.IdentityMapper;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.operators.DriverStrategy;
@@ -1426,10 +1426,10 @@ public class FeedbackPropertiesMatchTest {
}
private static final MapNode getMapNode() {
- return new MapNode(new MapOperatorBase<String, String, GenericMap<String,String>>(new IdentityMapper<String>(), new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), "map op"));
+ return new MapNode(new MapOperatorBase<String, String, MapFunction<String,String>>(new IdentityMapper<String>(), new UnaryOperatorInformation<String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), "map op"));
}
private static final MatchNode getJoinNode() {
- return new MatchNode(new JoinOperatorBase<String, String, String, GenericJoiner<String, String, String>>(new DummyJoinFunction<String>(), new BinaryOperatorInformation<String, String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[] {1}, new int[] {2}, "join op"));
+ return new MatchNode(new JoinOperatorBase<String, String, String, FlatJoinFunction<String, String, String>>(new DummyFlatJoinFunction<String>(), new BinaryOperatorInformation<String, String, String>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO), new int[] {1}, new int[] {2}, "join op"));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
index f3e513a..b9cc769 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
@@ -23,9 +23,9 @@ import java.util.Iterator;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.operators.util.FieldList;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.ReduceGroupOperator;
+import org.apache.flink.api.java.operators.GroupReduceOperator;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.util.Collector;
@@ -50,7 +50,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
- data.reduceGroup(new GroupReduceFunction<Double, Double>() {
+ data.reduceGroup(new RichGroupReduceFunction<Double, Double>() {
public void reduce(Iterator<Double> values, Collector<Double> out) {}
}).name("reducer")
.print().name("sink");
@@ -94,7 +94,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
- ReduceGroupOperator<Long, Long> reduced = data.reduceGroup(new GroupReduceFunction<Long, Long>() {
+ GroupReduceOperator<Long, Long> reduced = data.reduceGroup(new RichGroupReduceFunction<Long, Long>() {
public void reduce(Iterator<Long> values, Collector<Long> out) {}
}).name("reducer");
@@ -147,7 +147,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
data
.groupBy(1)
- .reduceGroup(new GroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer")
.print().name("sink");
@@ -194,9 +194,9 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
.name("source").setParallelism(6);
- ReduceGroupOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
+ GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
.groupBy(1)
- .reduceGroup(new GroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer");
@@ -255,7 +255,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
.groupBy(new KeySelector<Tuple2<String,Double>, String>() {
public String getKey(Tuple2<String, Double> value) { return value.f0; }
})
- .reduceGroup(new GroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer")
.print().name("sink");
@@ -309,11 +309,11 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
.name("source").setParallelism(6);
- ReduceGroupOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
+ GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
.groupBy(new KeySelector<Tuple2<String,Double>, String>() {
public String getKey(Tuple2<String, Double> value) { return value.f0; }
})
- .reduceGroup(new GroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
+ .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
index d3c3e3f..8fc4324 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.compiler;
import static org.junit.Assert.*;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.junit.Test;
import java.util.Iterator;
@@ -31,10 +32,9 @@ import org.apache.flink.api.java.DeltaIteration;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -261,7 +261,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
}
- public static final class Join222 extends JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class Join222 extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> join(Tuple2<Long, Long> vertexWithComponent, Tuple2<Long, Long> edge) {
@@ -269,13 +269,13 @@ public class IterationsCompilerTest extends CompilerTestBase {
}
}
- public static final class FlatMapJoin extends FlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
+ public static final class FlatMapJoin extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
@Override
public void flatMap(Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> value, Collector<Tuple2<Long, Long>> out) {}
}
- public static final class DummyMap extends MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class DummyMap extends RichMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception {
@@ -284,14 +284,14 @@ public class IterationsCompilerTest extends CompilerTestBase {
}
@ConstantFields("0")
- public static final class Reduce101 extends GroupReduceFunction<Tuple1<Long>, Tuple1<Long>> {
+ public static final class Reduce101 extends RichGroupReduceFunction<Tuple1<Long>, Tuple1<Long>> {
@Override
public void reduce(Iterator<Tuple1<Long>> values, Collector<Tuple1<Long>> out) {}
}
@ConstantFields("0")
- public static final class DuplicateValue extends MapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
+ public static final class DuplicateValue extends RichMapFunction<Tuple1<Long>, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> map(Tuple1<Long> value) throws Exception {
@@ -299,7 +299,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
}
}
- public static final class DuplicateValueScalar<T> extends MapFunction<T, Tuple2<T, T>> {
+ public static final class DuplicateValueScalar<T> extends RichMapFunction<T, Tuple2<T, T>> {
@Override
public Tuple2<T, T> map(T value) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/ReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/ReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/ReduceCompilationTest.java
index a654872..fb8ae8d 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/ReduceCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/ReduceCompilationTest.java
@@ -22,7 +22,7 @@ package org.apache.flink.compiler;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.junit.Test;
import org.apache.flink.api.java.DataSet;
@@ -46,7 +46,7 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
- data.reduce(new ReduceFunction<Double>() {
+ data.reduce(new RichReduceFunction<Double>() {
@Override
public Double reduce(Double value1, Double value2){
@@ -91,7 +91,7 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
- data.reduce(new ReduceFunction<Long>() {
+ data.reduce(new RichReduceFunction<Long>() {
@Override
public Long reduce(Long value1, Long value2){
@@ -145,7 +145,7 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
data
.groupBy(1)
- .reduce(new ReduceFunction<Tuple2<String,Double>>() {
+ .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
@Override
public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
return null;
@@ -205,7 +205,7 @@ public class ReduceCompilationTest extends CompilerTestBase implements java.io.S
.groupBy(new KeySelector<Tuple2<String,Double>, String>() {
public String getKey(Tuple2<String, Double> value) { return value.f0; }
})
- .reduce(new ReduceFunction<Tuple2<String,Double>>() {
+ .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
@Override
public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
return null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/UnionPropertyPropagationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/UnionPropertyPropagationTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/UnionPropertyPropagationTest.java
index 1f653c0..1020c8b 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/UnionPropertyPropagationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/UnionPropertyPropagationTest.java
@@ -25,7 +25,7 @@ import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.operators.base.FlatMapOperatorBase;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.apache.flink.api.java.record.operators.FileDataSink;
import org.apache.flink.api.java.record.operators.FileDataSource;
import org.apache.flink.api.java.record.operators.ReduceOperator;
@@ -174,7 +174,7 @@ public class UnionPropertyPropagationTest extends CompilerTestBase {
});
}
- public static final class DummyFlatMap extends FlatMapFunction<String, Tuple2<String, Integer>> {
+ public static final class DummyFlatMap extends RichFlatMapFunction<String, Tuple2<String, Integer>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
index e04256c..64a4791 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
@@ -31,9 +31,9 @@ import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.compiler.plan.DualInputPlanNode;
import org.apache.flink.compiler.plan.OptimizedPlan;
@@ -214,7 +214,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
iter.getWorkset().join(invariantInput)
.where(1, 2)
.equalTo(1, 2)
- .with(new JoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
+ .with(new RichJoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return first;
}
@@ -224,7 +224,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
result.join(iter.getSolutionSet())
.where(1, 0)
.equalTo(0, 2)
- .with(new JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
+ .with(new RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return second;
}
@@ -263,7 +263,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
iter.getWorkset().join(invariantInput)
.where(1, 2)
.equalTo(1, 2)
- .with(new JoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
+ .with(new RichJoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return first;
}
@@ -273,7 +273,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
.join(iter.getSolutionSet())
.where(1, 0)
.equalTo(1, 2)
- .with(new JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
+ .with(new RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return second;
}
@@ -282,7 +282,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
.withConstantSetSecond(joinPreservesSolutionSet ? new String[] {"0->0", "1->1", "2->2" } : null);
DataSet<Tuple3<Long, Long, Long>> nextWorkset = joinedWithSolutionSet.groupBy(1, 2)
- .reduceGroup(new GroupReduceFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>() {
+ .reduceGroup(new RichGroupReduceFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>() {
public void reduce(Iterator<Tuple3<Long, Long, Long>> values, Collector<Tuple3<Long, Long, Long>> out) {}
})
.name(NEXT_WORKSET_REDUCER_NAME)
@@ -290,7 +290,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
DataSet<Tuple3<Long, Long, Long>> nextSolutionSet = mapBeforeSolutionDelta ?
- joinedWithSolutionSet.map(new MapFunction<Tuple3<Long, Long, Long>,Tuple3<Long, Long, Long>>() { public Tuple3<Long, Long, Long> map(Tuple3<Long, Long, Long> value) { return value; } })
+ joinedWithSolutionSet.map(new RichMapFunction<Tuple3<Long, Long, Long>,Tuple3<Long, Long, Long>>() { public Tuple3<Long, Long, Long> map(Tuple3<Long, Long, Long> value) { return value; } })
.name(SOLUTION_DELTA_MAPPER_NAME).withConstantSet("0->0","1->1","2->2") :
joinedWithSolutionSet;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyFlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyFlatJoinFunction.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyFlatJoinFunction.java
new file mode 100644
index 0000000..2388db4
--- /dev/null
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyFlatJoinFunction.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.compiler.testfunctions;
+
+import org.apache.flink.api.java.functions.RichFlatJoinFunction;
+import org.apache.flink.util.Collector;
+
+public class DummyFlatJoinFunction<T> extends RichFlatJoinFunction<T, T, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void join(T first, T second, Collector<T> out) {
+ out.collect(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyJoinFunction.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyJoinFunction.java
deleted file mode 100644
index 0db075f..0000000
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/DummyJoinFunction.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.compiler.testfunctions;
-
-import org.apache.flink.api.java.functions.JoinFunction;
-
-public class DummyJoinFunction<T> extends JoinFunction<T, T, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T join(T first, T second) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
index fe61f25..42275af 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
@@ -21,13 +21,13 @@ package org.apache.flink.compiler.testfunctions;
import java.util.Iterator;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction.Combinable;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.util.Collector;
@Combinable
-public class IdentityGroupReducer<T> extends GroupReduceFunction<T, T> {
+public class IdentityGroupReducer<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityMapper.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityMapper.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityMapper.java
index b6aa40b..29fc2c8 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityMapper.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityMapper.java
@@ -19,9 +19,9 @@
package org.apache.flink.compiler.testfunctions;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
-public class IdentityMapper<T> extends MapFunction<T, T> {
+public class IdentityMapper<T> extends RichMapFunction<T, T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/SelectOneReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/SelectOneReducer.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/SelectOneReducer.java
index 91634cc..7ce267f 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/SelectOneReducer.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/SelectOneReducer.java
@@ -18,9 +18,9 @@
package org.apache.flink.compiler.testfunctions;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
-public class SelectOneReducer<T> extends ReduceFunction<T> {
+public class SelectOneReducer<T> extends RichReduceFunction<T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
index 26db00e..3f24e65 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
@@ -21,13 +21,13 @@ package org.apache.flink.compiler.testfunctions;
import java.util.Iterator;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction.Combinable;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.util.Collector;
@Combinable
-public class Top1GroupReducer<T> extends GroupReduceFunction<T, T> {
+public class Top1GroupReducer<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
index 51ad75d..736ee14 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCrossStub.java
@@ -23,14 +23,13 @@ import java.io.Serializable;
import org.apache.flink.api.java.record.functions.CrossFunction;
import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
-public class DummyCrossStub extends CrossFunction implements Serializable {
+public class DummyCrossStub extends CrossFunction {
private static final long serialVersionUID = 1L;
+
@Override
- public void cross(Record record1, Record record2, Collector<Record> out) {
- out.collect(record1);
- out.collect(record2);
+ public Record cross(Record first, Record second) throws Exception {
+ return first;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractFunction.java
deleted file mode 100644
index f4b2763..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractFunction.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import java.io.Serializable;
-
-import org.apache.flink.configuration.Configuration;
-
-/**
- * An abstract stub implementation for user-defined functions. It offers default implementations
- * for {@link #open(Configuration)} and {@link #close()}. It also offers access to the
- * {@link RuntimeContext} and {@link IterationRuntimeContext}.
- */
-public abstract class AbstractFunction implements Function, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- // --------------------------------------------------------------------------------------------
- // Runtime context access
- // --------------------------------------------------------------------------------------------
-
- private transient RuntimeContext runtimeContext;
-
- public void setRuntimeContext(RuntimeContext t) {
- if (this.runtimeContext == null) {
- this.runtimeContext = t;
- } else {
- throw new IllegalStateException("Error: The runtime context has already been set.");
- }
- }
-
- public RuntimeContext getRuntimeContext() {
- if (this.runtimeContext != null) {
- return this.runtimeContext;
- } else {
- throw new IllegalStateException("The runtime context has not been initialized.");
- }
- }
-
- public IterationRuntimeContext getIterationRuntimeContext() {
- if (this.runtimeContext == null) {
- throw new IllegalStateException("The runtime context has not been initialized.");
- } else if (this.runtimeContext instanceof IterationRuntimeContext) {
- return (IterationRuntimeContext) this.runtimeContext;
- } else {
- throw new IllegalStateException("This stub is not part of an iteration step function.");
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Default life cycle methods
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void open(Configuration parameters) throws Exception {}
-
- @Override
- public void close() throws Exception {}
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
new file mode 100644
index 0000000..07b957d
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/AbstractRichFunction.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * An abstract stub implementation for user-defined functions. It offers default implementations
+ * for {@link #open(Configuration)} and {@link #close()}. It also offers access to the
+ * {@link RuntimeContext} and {@link IterationRuntimeContext}.
+ */
+public abstract class AbstractRichFunction implements RichFunction, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // --------------------------------------------------------------------------------------------
+ // Runtime context access
+ // --------------------------------------------------------------------------------------------
+
+ private transient RuntimeContext runtimeContext;
+
+ public void setRuntimeContext(RuntimeContext t) {
+ if (this.runtimeContext == null) {
+ this.runtimeContext = t;
+ } else {
+ throw new IllegalStateException("Error: The runtime context has already been set.");
+ }
+ }
+
+ public RuntimeContext getRuntimeContext() {
+ if (this.runtimeContext != null) {
+ return this.runtimeContext;
+ } else {
+ throw new IllegalStateException("The runtime context has not been initialized.");
+ }
+ }
+
+ public IterationRuntimeContext getIterationRuntimeContext() {
+ if (this.runtimeContext == null) {
+ throw new IllegalStateException("The runtime context has not been initialized.");
+ } else if (this.runtimeContext instanceof IterationRuntimeContext) {
+ return (IterationRuntimeContext) this.runtimeContext;
+ } else {
+ throw new IllegalStateException("This stub is not part of an iteration step function.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Default life cycle methods
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public void open(Configuration parameters) throws Exception {}
+
+ @Override
+ public void close() throws Exception {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
new file mode 100644
index 0000000..5c200af
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.util.Collector;
+
+
+public interface CoGroupFunction<V1, V2, O> extends Function, Serializable {
+
+ /**
+ * This method must be implemented to provide a user implementation of a
+ * coGroup. It is called for each two key-value pairs that share the same
+ * key and come from different inputs.
+ *
+ * @param first The records from the first input which were paired with the key.
+ * @param second The records from the second input which were paired with the key.
+ * @param out A collector that collects all output pairs.
+ */
+ void coGroup(Iterator<V1> first, Iterator<V2> second, Collector<O> out) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
new file mode 100644
index 0000000..d72c4c8
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+/**
+ * Generic interface used for combiners.
+ */
+public interface CombineFunction<T> extends Function, Serializable {
+
+ T combine(Iterator<T> records) throws Exception;
+}
[04/39] git commit: Add notice to ClosureCleaner
Posted by rm...@apache.org.
Add notice to ClosureCleaner
[ci skip]
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/2e50eac5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/2e50eac5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/2e50eac5
Branch: refs/heads/travis_test
Commit: 2e50eac53dd2c41b4cf52a5aabc1b491608382e0
Parents: 911f673
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 24 01:33:48 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 24 01:36:18 2014 +0200
----------------------------------------------------------------------
.../org/apache/flink/api/scala/operators/ClosureCleaner.scala | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2e50eac5/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ClosureCleaner.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ClosureCleaner.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ClosureCleaner.scala
index 1b44bcf..228d87c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ClosureCleaner.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ClosureCleaner.scala
@@ -16,6 +16,11 @@
* limitations under the License.
*/
+/*
+ * NOTE: The code in this file is based on code from the Apache Spark
+ * project, licensed under the Apache License v 2.0
+ */
+
package org.apache.flink.api.scala.operators
import java.lang.reflect.Field
[36/39] git commit: Disable POJO types
Posted by rm...@apache.org.
Disable POJO types
There are various issues when working with POJOs in the Java API, for
example [1].
This commit squashes the following commmits:
1. Disable tests, which depend on POJO types
2. Replace expression keys with key selector in test
3. PackagedProgramEndToEndITCase used a KMeans variant, which relies on
expression keys to select the field to group on. This commit replaces
this with a key selector.
4. Disable WordCountPOJO test case and package
5. Disables WordCountPOJOITCase, which executes the WordCountPOJO example.
The example code is still included with the source code (with notice
that it is currently not working), but will *not* be packaged as a JAR
anymore.
6. Disable tests, which depend on POJO types (continued)
[1] https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E
This closes #93.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/1637cb12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/1637cb12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/1637cb12
Branch: refs/heads/travis_test
Commit: 1637cb12500cacfd4fdd9b0112c7af1683768e84
Parents: b6ae1ef
Author: uce <u....@fu-berlin.de>
Authored: Fri Aug 8 11:18:52 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Fri Aug 8 15:26:54 2014 +0200
----------------------------------------------------------------------
.../io/AvroInputFormatTypeExtractionTest.java | 38 ++++++++++----------
flink-examples/flink-java-examples/pom.xml | 8 +++--
.../example/java/wordcount/WordCountPOJO.java | 9 +++++
.../flink/api/java/typeutils/TypeExtractor.java | 10 +++---
.../api/java/operator/CoGroupOperatorTest.java | 7 +++-
.../flink/api/java/operator/GroupingTest.java | 5 +++
.../api/java/operator/JoinOperatorTest.java | 5 +++
.../type/extractor/PojoTypeInformationTest.java | 7 +++-
.../java/type/extractor/TypeExtractorTest.java | 12 +++----
.../typeutils/runtime/PojoSerializerTest.java | 32 +++++++++--------
.../WordCountPOJOITCase.java | 3 +-
.../PackagedProgramEndToEndITCase.java | 1 -
.../flink/test/util/testjar/KMeansForTest.java | 8 ++++-
13 files changed, 91 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
index aa08006..efe45fd 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
@@ -19,16 +19,15 @@
package org.apache.flink.api.java.io;
-import org.junit.Assert;
-import org.junit.Test;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.io.AvroInputFormat;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.core.fs.Path;
import org.apache.flink.types.TypeInformation;
+import org.junit.Assert;
+import org.junit.Test;
public class AvroInputFormatTypeExtractionTest {
@@ -36,44 +35,43 @@ public class AvroInputFormatTypeExtractionTest {
public void testTypeExtraction() {
try {
InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);
-
+
TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);
-
+
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<MyAvroType> input = env.createInput(format);
TypeInformation<?> typeInfoDataSet = input.getType();
-
-
- Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
- Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
-
+
+
+ Assert.assertTrue(typeInfoDirect instanceof GenericTypeInfo);
+ Assert.assertTrue(typeInfoDataSet instanceof GenericTypeInfo);
+
Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
- }
- catch (Exception e) {
+ } catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
-
+
public static final class MyAvroType {
-
+
public String theString;
-
+
private double aDouble;
-
+
public double getaDouble() {
return aDouble;
}
-
+
public void setaDouble(double aDouble) {
this.aDouble = aDouble;
}
-
+
public void setTheString(String theString) {
this.theString = theString;
}
-
+
public String getTheString() {
return theString;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-examples/flink-java-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/pom.xml b/flink-examples/flink-java-examples/pom.xml
index ea0db5d..97ebdec 100644
--- a/flink-examples/flink-java-examples/pom.xml
+++ b/flink-examples/flink-java-examples/pom.xml
@@ -291,7 +291,10 @@ under the License.
</configuration>
</execution>
- <!-- WordCountPOJO -->
+ <!-- WordCountPOJO
+
+ Note: disabled for now, because of problems with POJO types
+
<execution>
<id>WordCountPOJO</id>
<phase>package</phase>
@@ -313,8 +316,7 @@ under the License.
<include>**/java/wordcount/util/WordCountData.class</include>
</includes>
</configuration>
- </execution>
-
+ </execution> -->
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
index 04810a1..d993e60 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/wordcount/WordCountPOJO.java
@@ -53,6 +53,15 @@ public class WordCountPOJO {
// *************************************************************************
public static void main(String[] args) throws Exception {
+ // ====================================================================
+ // IMPORTANT
+ //
+ // Note: this example is currently not working, because support for
+ // POJO types has been disabled. As soon as all known issues (see [1])
+ // are fixed, we will enable POJO support again.
+ //
+ // [1] https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E
+ // ====================================================================
parseParameters(args);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index d03cc49..b596a87 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -792,11 +792,13 @@ public class TypeExtractor {
// special case handling for Class, this should not be handled by the POJO logic
return new GenericTypeInfo<X>(clazz);
}
- TypeInformation<X> pojoType = analyzePojo(clazz);
- if (pojoType != null) {
- return pojoType;
- }
+// Disable POJO types for now (see https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E)
+//
+// TypeInformation<X> pojoType = analyzePojo(clazz);
+// if (pojoType != null) {
+// return pojoType;
+// }
// return a generic type
return new GenericTypeInfo<X>(clazz);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
index 4bc7ac2..360050b 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.flink.api.java.DataSet;
@@ -128,6 +129,7 @@ public class CoGroupOperatorTest {
ds1.coGroup(ds2).where(5).equalTo(0);
}
+ @Ignore
@Test
public void testCoGroupKeyExpressions1() {
@@ -143,6 +145,7 @@ public class CoGroupOperatorTest {
}
}
+ @Ignore
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyExpressions2() {
@@ -154,6 +157,7 @@ public class CoGroupOperatorTest {
ds1.coGroup(ds2).where("myInt").equalTo("myString");
}
+ @Ignore
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyExpressions3() {
@@ -165,6 +169,7 @@ public class CoGroupOperatorTest {
ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
}
+ @Ignore
@Test(expected = IllegalArgumentException.class)
public void testCoGroupKeyExpressions4() {
@@ -175,7 +180,7 @@ public class CoGroupOperatorTest {
// should not work, cogroup key non-existent
ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
}
-
+
@Test
public void testCoGroupKeySelectors1() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
index 0938bd9..98b6998 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.flink.api.java.DataSet;
@@ -112,6 +113,7 @@ public class GroupingTest {
tupleDs.groupBy(-1);
}
+ @Ignore
@Test
public void testGroupByKeyExpressions1() {
@@ -129,6 +131,7 @@ public class GroupingTest {
}
}
+ @Ignore
@Test(expected = UnsupportedOperationException.class)
public void testGroupByKeyExpressions2() {
@@ -139,6 +142,7 @@ public class GroupingTest {
longDs.groupBy("myInt");
}
+ @Ignore
@Test(expected = InvalidProgramException.class)
public void testGroupByKeyExpressions3() {
@@ -152,6 +156,7 @@ public class GroupingTest {
}
+ @Ignore
@Test(expected = IllegalArgumentException.class)
public void testGroupByKeyExpressions4() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
index 8b88288..11eca83 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.apache.flink.api.java.DataSet;
@@ -128,6 +129,7 @@ public class JoinOperatorTest {
ds1.join(ds2).where(5).equalTo(0);
}
+ @Ignore
@Test
public void testJoinKeyExpressions1() {
@@ -143,6 +145,7 @@ public class JoinOperatorTest {
}
}
+ @Ignore
@Test(expected = InvalidProgramException.class)
public void testJoinKeyExpressions2() {
@@ -154,6 +157,7 @@ public class JoinOperatorTest {
ds1.join(ds2).where("myInt").equalTo("myString");
}
+ @Ignore
@Test(expected = InvalidProgramException.class)
public void testJoinKeyExpressions3() {
@@ -165,6 +169,7 @@ public class JoinOperatorTest {
ds1.join(ds2).where("myInt", "myString").equalTo("myString");
}
+ @Ignore
@Test(expected = IllegalArgumentException.class)
public void testJoinKeyExpressions4() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
index b963574..cba8e88 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.types.TypeInformation;
+import org.junit.Ignore;
import org.junit.Test;
@SuppressWarnings("unused")
@@ -51,6 +52,7 @@ public class PojoTypeInformationTest {
Integer[] intWrapperArray;
}
+ @Ignore
@Test
public void testSimplePojoTypeExtraction() {
TypeInformation<SimplePojo> type = TypeExtractor.getForClass(SimplePojo.class);
@@ -66,6 +68,7 @@ public class PojoTypeInformationTest {
NestedPojoInner inner;
}
+ @Ignore
@Test
public void testNestedPojoTypeExtraction() {
TypeInformation<NestedPojoOuter> type = TypeExtractor.getForClass(NestedPojoOuter.class);
@@ -82,6 +85,7 @@ public class PojoTypeInformationTest {
Recursive1Pojo rec;
}
+ @Ignore
@Test
public void testRecursivePojoTypeExtraction() {
// This one tests whether a recursive pojo is detected using the set of visited
@@ -89,7 +93,8 @@ public class PojoTypeInformationTest {
TypeInformation<Recursive1Pojo> type = TypeExtractor.getForClass(Recursive1Pojo.class);
assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
}
-
+
+ @Ignore
@Test
public void testRecursivePojoObjectTypeExtraction() {
TypeInformation<Recursive1Pojo> type = TypeExtractor.getForObject(new Recursive1Pojo());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index c8a8ee9..412f751 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -41,7 +41,6 @@ import org.apache.flink.api.java.typeutils.BasicArrayTypeInfo;
import org.apache.flink.api.java.typeutils.BasicTypeInfo;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.PrimitiveArrayTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -62,7 +61,6 @@ import org.junit.Test;
public class TypeExtractorTest {
-
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testBasicType() {
@@ -307,11 +305,11 @@ public class TypeExtractorTest {
Assert.assertFalse(ti.isBasicType());
Assert.assertFalse(ti.isTupleType());
- Assert.assertTrue(ti instanceof PojoTypeInfo);
+ Assert.assertTrue(ti instanceof GenericTypeInfo);
Assert.assertEquals(ti.getTypeClass(), CustomType.class);
// use getForClass()
- Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof PojoTypeInfo);
+ Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof GenericTypeInfo);
Assert.assertEquals(TypeExtractor.getForClass(CustomType.class).getTypeClass(), ti.getTypeClass());
// use getForObject()
@@ -320,7 +318,7 @@ public class TypeExtractorTest {
Assert.assertFalse(ti2.isBasicType());
Assert.assertFalse(ti2.isTupleType());
- Assert.assertTrue(ti2 instanceof PojoTypeInfo);
+ Assert.assertTrue(ti2 instanceof GenericTypeInfo);
Assert.assertEquals(ti2.getTypeClass(), CustomType.class);
}
@@ -360,7 +358,7 @@ public class TypeExtractorTest {
Assert.assertEquals(Tuple2.class, tti.getTypeClass());
Assert.assertEquals(Long.class, tti.getTypeAt(0).getTypeClass());
- Assert.assertTrue(tti.getTypeAt(1) instanceof PojoTypeInfo);
+ Assert.assertTrue(tti.getTypeAt(1) instanceof GenericTypeInfo);
Assert.assertEquals(CustomType.class, tti.getTypeAt(1).getTypeClass());
// use getForObject()
@@ -373,7 +371,7 @@ public class TypeExtractorTest {
Assert.assertEquals(Tuple2.class, tti2.getTypeClass());
Assert.assertEquals(Long.class, tti2.getTypeAt(0).getTypeClass());
- Assert.assertTrue(tti2.getTypeAt(1) instanceof PojoTypeInfo);
+ Assert.assertTrue(tti2.getTypeAt(1) instanceof GenericTypeInfo);
Assert.assertEquals(CustomType.class, tti2.getTypeAt(1).getTypeClass());
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index 5aef84e..2d17764 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -18,48 +18,48 @@
package org.apache.flink.api.java.typeutils.runtime;
-import java.util.Random;
-
+import com.google.common.base.Objects;
import org.apache.flink.api.common.typeutils.SerializerTestBase;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
import org.apache.flink.types.TypeInformation;
+import org.junit.Ignore;
-import com.google.common.base.Objects;
+import java.util.Random;
/**
* A test for the {@link org.apache.flink.api.java.typeutils.runtime.PojoSerializer}.
*/
+@Ignore
public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.TestUserClass> {
private TypeInformation<TestUserClass> type = TypeExtractor.getForClass(TestUserClass.class);
@Override
protected TypeSerializer<TestUserClass> createSerializer() {
TypeSerializer<TestUserClass> serializer = type.createSerializer();
- assert(serializer instanceof PojoSerializer);
+ assert (serializer instanceof PojoSerializer);
return serializer;
}
-
+
@Override
protected int getLength() {
return -1;
}
-
+
@Override
protected Class<TestUserClass> getTypeClass() {
return TestUserClass.class;
}
-
+
@Override
protected TestUserClass[] getTestData() {
Random rnd = new Random(874597969123412341L);
- return new TestUserClass[] {
- new TestUserClass(rnd.nextInt(), "foo", rnd.nextDouble(), new int[] {1,2,3},
- new NestedTestUserClass(rnd.nextInt(), "foo@boo", rnd.nextDouble(), new int[] {10, 11, 12})),
- new TestUserClass(rnd.nextInt(), "bar", rnd.nextDouble(), new int[] {4,5,6},
- new NestedTestUserClass(rnd.nextInt(), "bar@bas", rnd.nextDouble(), new int[] {20, 21, 22}))
+ return new TestUserClass[]{
+ new TestUserClass(rnd.nextInt(), "foo", rnd.nextDouble(), new int[]{1, 2, 3},
+ new NestedTestUserClass(rnd.nextInt(), "foo@boo", rnd.nextDouble(), new int[]{10, 11, 12})),
+ new TestUserClass(rnd.nextInt(), "bar", rnd.nextDouble(), new int[]{4, 5, 6},
+ new NestedTestUserClass(rnd.nextInt(), "bar@bas", rnd.nextDouble(), new int[]{20, 21, 22}))
};
}
@@ -73,7 +73,8 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
private NestedTestUserClass nestedClass;
- public TestUserClass() {}
+ public TestUserClass() {
+ }
public TestUserClass(int dumm1, String dumm2, double dumm3, int[] dumm4, NestedTestUserClass nestedClass) {
this.dumm1 = dumm1;
@@ -124,7 +125,8 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
public double dumm3;
private int[] dumm4;
- public NestedTestUserClass() {}
+ public NestedTestUserClass() {
+ }
public NestedTestUserClass(int dumm1, String dumm2, double dumm3, int[] dumm4) {
this.dumm1 = dumm1;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
index f463924..7e16fd4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
@@ -21,8 +21,9 @@ package org.apache.flink.test.exampleJavaPrograms;
import org.apache.flink.example.java.wordcount.WordCountPOJO;
import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.JavaProgramTestBase;
+import org.junit.Ignore;
-
+@Ignore
public class WordCountPOJOITCase extends JavaProgramTestBase {
protected String textPath;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java b/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
index 0e93f2a..d244811 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/localDistributed/PackagedProgramEndToEndITCase.java
@@ -64,7 +64,6 @@ public class PackagedProgramEndToEndITCase {
fwClusters.close();
String jarPath = "target/maven-test-jar.jar";
-// String jarPath = "/home/aljoscha/maven-test-jar.jar";
// run KMeans
cluster.setNumTaskTracker(2);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1637cb12/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
index 7149cd3..917615d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.Program;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -79,7 +80,12 @@ public class KMeansForTest implements Program {
.map(new SelectNearestCenter()).withBroadcastSet(loop, "centroids")
// count and sum point coordinates for each centroid
.map(new CountAppender())
- .groupBy("f0").reduce(new CentroidAccumulator())
+ .groupBy(new KeySelector<DummyTuple3IntPointLong, Integer>() {
+ @Override
+ public Integer getKey(DummyTuple3IntPointLong value) throws Exception {
+ return value.f0;
+ }
+ }).reduce(new CentroidAccumulator())
// compute new centroids from point counts and coordinate sums
.map(new CentroidAverager());
[33/39] git commit: Reflect recent changes to Java API documentation
Posted by rm...@apache.org.
Reflect recent changes to Java API documentation
change text to String
This closes #87.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/71cccd6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/71cccd6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/71cccd6d
Branch: refs/heads/travis_test
Commit: 71cccd6d88fcefd824960d312153e2df523beb4d
Parents: ecb829e
Author: Kostas Tzoumas <Ko...@gmail.com>
Authored: Thu Jul 31 21:21:35 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Aug 6 11:46:45 2014 +0200
----------------------------------------------------------------------
docs/java_api_guide.md | 214 +++++++++++++++++++++++++++-------
docs/java_api_transformations.md | 29 ++++-
2 files changed, 202 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/71cccd6d/docs/java_api_guide.md
----------------------------------------------------------------------
diff --git a/docs/java_api_guide.md b/docs/java_api_guide.md
index f22b2b7..76dd332 100644
--- a/docs/java_api_guide.md
+++ b/docs/java_api_guide.md
@@ -31,14 +31,14 @@ public class WordCountExample {
public static void main(String[] args) throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- DataSet<String> text = env.fromElements(
+ DataSet<String> text = env.fromElements(
"Who's there?",
"I think I hear them. Stand, ho! Who's there?");
DataSet<Tuple2<String, Integer>> wordCounts = text
.flatMap(new LineSplitter())
.groupBy(0)
- .aggregate(Aggregations.SUM, 1);
+ .sum(1);
wordCounts.print();
@@ -140,7 +140,7 @@ an execution environment for executing your program on a cluster.
For specifying data sources the execution environment has several methods
to read from files using various methods: you can just read them line by line,
as CSV files, or using completely custom data input formats. To just read
-a text file as a sequence of lines, you could use:
+a text file as a sequence of lines, you can use:
```java
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@@ -156,7 +156,7 @@ Once you have a `DataSet` you can apply transformations to create a new
`DataSet` which you can then write to a file, transform again, or
combine with other `DataSet`s. You apply transformations by calling
methods on `DataSet` with your own custom transformation function. For example,
-map looks like this:
+a map transformation looks like this:
```java
DataSet<String> input = ...;
@@ -262,7 +262,7 @@ data.flatMap(new FlatMapFunction<String, String>() {
<tr>
<td><strong>Filter</strong></td>
<td>
- <p>Evaluates a boolean function for each element and retains those for which the function returns *true*.</p>
+ <p>Evaluates a boolean function for each element and retains those for which the function returns true.</p>
{% highlight java %}
data.filter(new FilterFunction<Integer>() {
public boolean filter(Integer value) { return value > 1000; }
@@ -307,45 +307,21 @@ data.reduceGroup(new GroupReduceFunction<Integer, Integer> {
<td>
<p>Aggregates a group of values into a single value. Aggregation functions can be thought of as built-in reduce functions. Aggregate may be applied on a full data set, or on a grouped data set.</p>
{% highlight java %}
-DataSet<Tuple3<Integer, String, Double>> input = // [...]
-DataSet<Tuple3<Integer, String, Double>> output = input.aggregate(SUM, 0).and(MIN, 2);
-{% endhighlight %}
- </td>
- </tr>
-
- <tr>
- <td><strong>ReduceGroup</strong></td>
- <td>
- <p>Combines a group of elements into one or more elements. ReduceGroup may be applied on a full data set, or on a grouped data set.</p>
-{% highlight java %}
-data.reduceGroup(new GroupReduceFunction<Integer, Integer> {
- public void reduceGroup(Iterable<Integer> values, Collector<Integer> out) {
- int prefixSum = 0;
- for (Integer i : values) {
- prefixSum += i;
- out.collect(prefixSum);
- }
- }
-});
-{% endhighlight %}
- </td>
- </tr>
-
- <tr>
- <td><strong>Aggregate</strong></td>
- <td>
- <p>Aggregates a group of values into a single value. Aggregation functions can be thought of as built-in reduce functions. Aggregate may be applied on a full data set, or on a grouped data set.</p>
-{% highlight java %}
-DataSet<Tuple3<Integer, String, Double>> input = // [...]
+Dataset<Tuple3<Integer, String, Double>> input = // [...]
DataSet<Tuple3<Integer, String, Double>> output = input.aggregate(SUM, 0).and(MIN, 2);
{% endhighlight %}
+ <p>You can also use short-hand syntax for minimum, maximum, and sum aggregations.</p>
+ {% highlight java %}
+ Dataset<Tuple3<Integer, String, Double>> input = // [...]
+DataSet<Tuple3<Integer, String, Double>> output = input.sum(0).andMin(2);
+ {% endhighlight %}
</td>
</tr>
</tr>
<td><strong>Join</strong></td>
<td>
- <p>Joins two data sets by creating all pairs of elements that are equal on their keys. Optionally uses a JoinFunction to turn the pair of elements into a single element. See [keys](#keys) on how to define join keys.</p>
+ Joins two data sets by creating all pairs of elements that are equal on their keys. Optionally uses a JoinFunction to turn the pair of elements into a single element, or a FlatJoinFunction to turn the pair of elements into arbitararily many (including none) elements. See [keys](#keys) on how to define join keys.
{% highlight java %}
result = input1.join(input2)
.where(0) // key of the first input (tuple field 0)
@@ -374,7 +350,7 @@ data1.coGroup(data2)
<tr>
<td><strong>Cross</strong></td>
<td>
- <p>Builds the cartesian product (cross product) of two inputs, creating all pairs of elements. Optionally uses a CrossFunction to turn the pair of elements into a single element</p>
+ <p>Builds the Cartesian product (cross product) of two inputs, creating all pairs of elements. Optionally uses a CrossFunction to turn the pair of elements into a single element</p>
{% highlight java %}
DataSet<Integer> data1 = // [...]
DataSet<String> data2 = // [...]
@@ -427,6 +403,88 @@ DataSet<Tuple2<String, Integer>> out = in.project(2,0).types(String.class, Integ
Defining Keys
-------------
+One transformation (join, coGroup) require that a key is defined on
+its argument DataSets, and other transformations (Reduce, GroupReduce,
+Aggregate) allow that the DataSet is grouped on a key before they are
+applied.
+
+A DataSet is grouped as
+{% highlight java %}
+DataSet<...> input = // [...]
+DataSet<...> reduced = input
+ .groupBy(/*define key here*/)
+ .reduceGroup(/*do something*/);
+{% endhighlight %}
+
+The data model of Flink is not based on key-value pairs. Therefore,
+you do not need to physically pack the data set types into keys and
+values. Keys are "virtual": they are defined as functions over the
+actual data to guide the grouping operator.
+
+The simplest case is grouping a data set of Tuples on one or more
+fields of the Tuple:
+{% highlight java %}
+DataSet<Tuple3<Integer,String,Long>> input = // [...]
+DataSet<Tuple3<Integer,String,Long> grouped = input
+ .groupBy(0)
+ .reduceGroup(/*do something*/);
+{% endhighlight %}
+
+The data set is grouped on the first field of the tuples (the one of
+Integer type). The GroupReduceFunction will thus receive groups with
+the same value of the first field.
+
+{% highlight java %}
+DataSet<Tuple3<Integer,String,Long>> input = // [...]
+DataSet<Tuple3<Integer,String,Long> grouped = input
+ .groupBy(0,1)
+ .reduce(/*do something*/);
+{% endhighlight %}
+
+The data set is grouped on the composite key consisting of the first and the
+second fields, therefore the GroupReduceFuntion will receive groups
+with the same value for both fields.
+
+In general, key definition is done via a "key selector" function, which
+takes as argument one dataset element and returns a key of an
+arbitrary data type by performing an arbitrary computation on this
+element. For example:
+{% highlight java %}
+// some ordinary POJO
+public class WC {public String word; public int count;}
+DataSet<WC> words = // [...]
+DataSet<WC> wordCounts = words
+ .groupBy(
+ new KeySelector<WC, String>() {
+ public String getKey(WC wc) { return wc.word; }
+ })
+ .reduce(/*do something*/);
+{% endhighlight %}
+
+Remember that keys are not only used for grouping, but also joining and matching data sets:
+{% highlight java %}
+// some POJO
+public class Rating {
+ public String name;
+ public String category;
+ public int points;
+}
+DataSet<Rating> ratings = // [...]
+DataSet<Tuple2<String, Double>> weights = // [...]
+DataSet<Tuple2<String, Double>>
+ weightedRatings =
+ ratings.join(weights)
+
+ // key of the first input
+ .where(new KeySelector<Rating, String>() {
+ public String getKey(Rating r) { return r.category; }
+ })
+
+ // key of the second input
+ .equalTo(new KeySelector<Tuple2<String, Double>, String>() {
+ public String getKey(Tuple2<String, Double> t) { return t.f0; }
+ });
+{% endhighlight %}
[Back to top](#top)
@@ -435,6 +493,85 @@ Defining Keys
Functions
---------
+You can define a user-defined function and pass it to the DataSet
+transformations in several ways:
+
+#### Implementing an interface
+
+The most basic way is to implement one of the provided interfaces:
+
+{% highlight java %}
+class MyMapFunction implements MapFunction<String, Integer> {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+data.map (new MyMapFunction());
+{% endhighlight %}
+
+#### Anonymous classes
+
+You can pass a function as an anonmymous class:
+{% highlight java %}
+data.map(new MapFunction<String, Integer> () {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+{% endhighlight %}
+
+#### Java 8 Lambdas
+
+***Warning: Lambdas are currently only supported for filter and reduce
+ transformations***
+
+{% highlight java %}
+DataSet<String> data = // [...]
+data.filter(s -> s.startsWith("http://"));
+{% endhighlight %}
+
+{% highlight java %}
+DataSet<Integer> data = // [...]
+data.reduce((i1,i2) -> i1 + i2);
+{% endhighlight %}
+
+#### Rich functions
+
+All transformations that take as argument a user-defined function can
+instead take as argument a *rich* function. For example, instead of
+{% highlight java %}
+class MyMapFunction implements MapFunction<String, Integer> {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+{% endhighlight %}
+you can write
+{% highlight java %}
+class MyMapFunction extends RichMapFunction<String, Integer> {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+{% endhighlight %}
+and pass the function as usual to a `map` transformation:
+{% highlight java %}
+data.map(new MyMapFunction());
+{% endhighlight %}
+
+Rich functions can also be defined as an anonymous class:
+{% highlight java %}
+data.map (new RichMapFunction<String, Integer>() {
+ public Integer map(String value) { return Integer.parseInt(value); }
+});
+{% endhighlight %}
+
+Rich functions provide, in addition to the user-defined function (map,
+reduce, etc), four methods: `open`, `close`, `getRuntimeContext`, and
+`setRuntimeContext`. These are useful for creating and finalizing
+local state, accessing broadcast variables (see
+[Broadcast Variables](#broadcast_variables), and for accessing runtime
+information such as accumulators and counters (see
+[Accumulators and Counters](#accumulators_counters), and information
+on iterations (see [Iterations](#iterations)).
+
+In particular for the `reduceGroup` transformation, using a rich
+function is the only way to define an optional `combine` function. See
+the
+[transformations documentation]({{site.baseurl}}/java_api_transformations.html)
+for a complete example.
[Back to top](#top)
@@ -502,7 +639,7 @@ wordCounts.map(new MapFunction<WordCount, Integer>() {
When working with operators that require a Key for grouping or matching records
you need to implement a `KeySelector` for your custom type (see
-[Section Data Transformations](#transformations)).
+[Section Defining Keys](#keys)).
```java
wordCounts.groupBy(new KeySelector<WordCount, String>() {
@@ -1043,4 +1180,3 @@ The script to start the webinterface is located under ```bin/start-webclient.sh`
You are able to specify program arguments in the textbox at the bottom of the page. Checking the plan visualization checkbox shows the execution plan before executing the actual program.
[Back to top](#top)
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/71cccd6d/docs/java_api_transformations.md
----------------------------------------------------------------------
diff --git a/docs/java_api_transformations.md b/docs/java_api_transformations.md
index bd5c45b..e4980de 100644
--- a/docs/java_api_transformations.md
+++ b/docs/java_api_transformations.md
@@ -241,7 +241,11 @@ DataSet<Double> output = input
#### Combinable GroupReduceFunctions
-In contrast to a `ReduceFunction`, a `GroupReduceFunction` is not necessarily combinable. In order to make a `GroupReduceFunction` combinable, you need to implement (override) the `combine()` method and annotate the `GroupReduceFunction` with the `@Combinable` annotation as shown here:
+In contrast to a `ReduceFunction`, a `GroupReduceFunction` is not
+necessarily combinable. In order to make a `GroupReduceFunction`
+combinable, you need to use the `RichGroupReduceFunction` variant,
+implement (override) the `combine()` method, and annotate the
+`GroupReduceFunction` with the `@Combinable` annotation as shown here:
```java
// Combinable GroupReduceFunction that computes two sums.
@@ -431,6 +435,28 @@ DataSet<Tuple2<String, Double>>
.with(new PointWeighter());
```
+#### Join with FlatJoinFunction
+
+Analogous to Map and FlatMap, a FlatJoin function behaves in the same
+way as a JoinFunction, but instead of returning one element, it can
+return (collect), zero, one, or more elements.
+{% highlight java %}
+public class PointWeighter
+ implements FlatJoinFunction<Rating, Tuple2<String, Double>, Tuple2<String, Double>> {
+ @Override
+ public void join(Rating rating, Tuple2<String, Double> weight,
+ Collector<Tuple2<String, Double>> out) {
+ if (weight.f1 > 0.1) {
+ out.collect(new Tuple2<String, Double>(rating.name, rating.points * weight.f1));
+ }
+ }
+}
+
+DataSet<Tuple2<String, Double>>
+ weightedRatings =
+ ratings.join(weights) // [...]
+{% endhighlight %}
+
#### Join with Projection
A Join transformation can construct result tuples using a projection as shown here:
@@ -625,4 +651,3 @@ DataSet<Tuple2<String, Integer>> unioned = vals1.union(vals2)
[Back to top](#top)
-
[08/39] git commit: [FLINK-998] Close network connections when idle
Posted by rm...@apache.org.
[FLINK-998] Close network connections when idle
1. Close idle TCP connections
Tasks enqueue envelopes to be dispatched via NettyConnectionManager. The
first task to enqueue an envelope triggers the establishment of a new
TCP connection to the target task manager. Usually, there are multiple
connections between the same task managers in order to avoid deadlocks.
This commit addresses the following problem: idle connections were never
closed. This meant that the number of TCP connections was monotonically
increasing over time. Idling connections are now closed after a
configured amount of time, see 'taskmanager.net.closeAfterIdleForMs'.
2. Enqueue envelopes directly
TCP connections are handled by Netty's IO loop. Every enqueue operation
needs to ensure that the envelope to be enqueued is handed over to the
respective IO thread in a thread-safe manner. Previously, this was done
via Netty's user events.
This commit addresses the following problem: every enqueue operation
resulted in the creation of an IO event loop task to hand over the
envelope. Envelopes are now directly added to a ConcurrentLinkedQueue,
which drastically improves the throughput of enqueue operations. The
user event is now only fired to trigger the processing if the queue was
empty.
3. Added unit tests for OutboundConnectionQueue and adapted existing
tests to test concurrent enqueue and close
4. Removed unused arguments to NettyConnectionManager (low and high
water mark)
This closes #76.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/52512636
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/52512636
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/52512636
Branch: refs/heads/travis_test
Commit: 52512636444902497e47ccbfb1cabaffb3e23343
Parents: bef5e54
Author: uce <u....@fu-berlin.de>
Authored: Thu Jul 17 16:32:06 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Wed Jul 30 01:03:07 2014 +0200
----------------------------------------------------------------------
.../flink/configuration/ConfigConstants.java | 20 +-
.../io/network/LocalConnectionManager.java | 9 +-
.../io/network/NetworkConnectionManager.java | 7 +-
.../runtime/io/network/RemoteReceiver.java | 2 +-
.../network/netty/NettyConnectionManager.java | 108 ++--
.../network/netty/OutboundConnectionQueue.java | 164 +++--
.../flink/runtime/taskmanager/TaskManager.java | 13 +-
.../netty/NettyConnectionManagerTest.java | 272 +++++----
.../netty/OutboundConnectionQueueTest.java | 606 +++++++++++++++++++
9 files changed, 988 insertions(+), 213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index 0c0800a..20a7019 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -117,14 +117,9 @@ public final class ConfigConstants {
public static final String TASK_MANAGER_NET_NUM_OUT_THREADS_KEY = "taskmanager.net.numOutThreads";
/**
- * The low water mark used in NettyConnectionManager for the Bootstrap.
+ * The minimum time in ms a channel must be idle, before it will be closed.
*/
- public static final String TASK_MANAGER_NET_NETTY_LOW_WATER_MARK = "taskmanager.net.nettyLowWaterMark";
-
- /**
- * The high water mark used in NettyConnectionManager for the Bootstrap.
- */
- public static final String TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK = "taskmanager.net.nettyHighWaterMark";
+ public static final String TASK_MANAGER_NET_CLOSE_AFTER_IDLE_FOR_MS_KEY = "taskmanager.net.closeAfterIdleForMs";
/**
* Parameter for the interval in which the TaskManager sends the periodic heart beat messages
@@ -354,16 +349,9 @@ public final class ConfigConstants {
public static final int DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS = -1;
/**
- * Default low water mark used in NettyConnectionManager for the Bootstrap. If set to -1, NettyConnectionManager
- * will use half of the network buffer size as the low water mark.
- */
- public static final int DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK = -1;
-
- /**
- * Default high water mark used in NettyConnectionManager for the Bootstrap. If set to -1, NettyConnectionManager
- * will use the network buffer size as the high water mark.
+ * The minimum time in ms a channel must be idle, before it will be closed.
*/
- public static final int DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK = -1;
+ public static final int DEFAULT_TASK_MANAGER_NET_CLOSE_AFTER_IDLE_FOR_MS = 10000;
/**
* The default interval for TaskManager heart beats (2000 msecs).
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
index 645f71e..612d6d5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/LocalConnectionManager.java
@@ -16,11 +16,14 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.io.network;
import java.io.IOException;
+/**
+ * Local dummy network connection manager used, when the task manager
+ * runs in local mode without starting up the remote network stack.
+ */
public class LocalConnectionManager implements NetworkConnectionManager {
@Override
@@ -32,6 +35,10 @@ public class LocalConnectionManager implements NetworkConnectionManager {
}
@Override
+ public void close(RemoteReceiver receiver) {
+ }
+
+ @Override
public void shutdown() throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkConnectionManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkConnectionManager.java
index 7e3e0b1..ceef2c7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkConnectionManager.java
@@ -16,16 +16,21 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.io.network;
import java.io.IOException;
+/**
+ * The network connection manager is responsible to dispatch envelopes
+ * to remote receivers.
+ */
public interface NetworkConnectionManager {
public void start(ChannelManager channelManager) throws IOException;
public void enqueue(Envelope envelope, RemoteReceiver receiver) throws IOException;
+ public void close(RemoteReceiver receiver);
+
public void shutdown() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/RemoteReceiver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/RemoteReceiver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/RemoteReceiver.java
index 23d1205..8815ea5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/RemoteReceiver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/RemoteReceiver.java
@@ -31,7 +31,7 @@ import org.apache.flink.core.memory.DataOutputView;
* Objects of this class uniquely identify a connection to a remote {@link TaskManager}.
*
*/
-public final class RemoteReceiver implements IOReadableWritable {
+public class RemoteReceiver implements IOReadableWritable {
/**
* The address of the connection to the remote {@link TaskManager}.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
index 073893e..7e9af6f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManager.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.io.network.netty;
import io.netty.bootstrap.Bootstrap;
@@ -31,7 +30,6 @@ import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.runtime.io.network.ChannelManager;
@@ -70,12 +68,19 @@ public class NettyConnectionManager implements NetworkConnectionManager {
private final int highWaterMark;
+ private final int closeAfterIdleForMs;
+
private ServerBootstrap in;
private Bootstrap out;
- public NettyConnectionManager(InetAddress bindAddress, int bindPort, int bufferSize, int numInThreads,
- int numOutThreads, int lowWaterMark, int highWaterMark) {
+ public NettyConnectionManager(
+ InetAddress bindAddress,
+ int bindPort,
+ int bufferSize,
+ int numInThreads,
+ int numOutThreads,
+ int closeAfterIdleForMs) {
this.bindAddress = bindAddress;
this.bindPort = bindPort;
@@ -87,14 +92,19 @@ public class NettyConnectionManager implements NetworkConnectionManager {
this.numInThreads = (numInThreads == -1) ? defaultNumThreads : numInThreads;
this.numOutThreads = (numOutThreads == -1) ? defaultNumThreads : numOutThreads;
- this.lowWaterMark = (lowWaterMark == -1) ? bufferSize / 2 : lowWaterMark;
- this.highWaterMark = (highWaterMark == -1) ? bufferSize : highWaterMark;
+ this.lowWaterMark = bufferSize / 2;
+ this.highWaterMark = bufferSize;
+
+ this.closeAfterIdleForMs = closeAfterIdleForMs;
}
@Override
public void start(ChannelManager channelManager) throws IOException {
- LOG.info(String.format("Starting with %d incoming and %d outgoing connection threads.", numInThreads, numOutThreads));
- LOG.info(String.format("Setting low water mark to %d and high water mark to %d bytes.", lowWaterMark, highWaterMark));
+ LOG.info(String.format("Starting with %d incoming and %d outgoing connection threads.",
+ numInThreads, numOutThreads));
+ LOG.info(String.format("Setting low water mark to %d and high water mark to %d bytes.",
+ lowWaterMark, highWaterMark));
+ LOG.info(String.format("Close channels after idle for %d ms.", closeAfterIdleForMs));
final BufferProviderBroker bufferProviderBroker = channelManager;
final EnvelopeDispatcher envelopeDispatcher = channelManager;
@@ -102,16 +112,16 @@ public class NettyConnectionManager implements NetworkConnectionManager {
int numHeapArenas = 0;
int numDirectArenas = numInThreads + numOutThreads;
int pageSize = bufferSize << 1;
- int chunkSize = 16 * 1 << 20; // 16 MB
+ int chunkSize = 16 << 20; // 16 MB
// shift pageSize maxOrder times to get to chunkSize
- int maxOrder = (int) (Math.log(chunkSize/pageSize) / Math.log(2));
+ int maxOrder = (int) (Math.log(chunkSize / pageSize) / Math.log(2));
PooledByteBufAllocator pooledByteBufAllocator =
new PooledByteBufAllocator(true, numHeapArenas, numDirectArenas, pageSize, maxOrder);
String msg = String.format("Instantiated PooledByteBufAllocator with direct arenas: %d, heap arenas: %d, " +
- "page size (bytes): %d, chunk size (bytes): %d.",
+ "page size (bytes): %d, chunk size (bytes): %d.",
numDirectArenas, numHeapArenas, pageSize, (pageSize << maxOrder));
LOG.info(msg);
@@ -188,7 +198,7 @@ public class NettyConnectionManager implements NetworkConnectionManager {
// 2) a channel that is in buildup (sometimes) -> attach to the future and wait for the actual channel
// 3) not yet existing -> establish the channel
- final Object entry = this.outConnections.get(receiver);
+ final Object entry = outConnections.get(receiver);
final OutboundConnectionQueue channel;
if (entry != null) {
@@ -206,14 +216,14 @@ public class NettyConnectionManager implements NetworkConnectionManager {
// We create a "buildup future" and atomically add it to the map.
// Only the thread that really added it establishes the channel.
// The others need to wait on that original establisher's future.
- ChannelInBuildup inBuildup = new ChannelInBuildup(this.out, receiver);
- Object old = this.outConnections.putIfAbsent(receiver, inBuildup);
+ ChannelInBuildup inBuildup = new ChannelInBuildup(out, receiver, this, closeAfterIdleForMs);
+ Object old = outConnections.putIfAbsent(receiver, inBuildup);
if (old == null) {
- this.out.connect(receiver.getConnectionAddress()).addListener(inBuildup);
+ out.connect(receiver.getConnectionAddress()).addListener(inBuildup);
channel = inBuildup.waitForChannel();
- Object previous = this.outConnections.put(receiver, channel);
+ Object previous = outConnections.put(receiver, channel);
if (inBuildup != previous) {
throw new IOException("Race condition during channel build up.");
@@ -227,7 +237,19 @@ public class NettyConnectionManager implements NetworkConnectionManager {
}
}
- channel.enqueue(envelope);
+ if (!channel.enqueue(envelope)) {
+ // The channel has been closed, try again.
+ LOG.debug("Retry enqueue on channel: " + channel + ".");
+
+ // This will either establish a new connection or use the
+ // one, which has been established in the mean time.
+ enqueue(envelope, receiver);
+ }
+ }
+
+ @Override
+ public void close(RemoteReceiver receiver) {
+ outConnections.remove(receiver);
}
@Override
@@ -256,9 +278,9 @@ public class NettyConnectionManager implements NetworkConnectionManager {
private String getNonZeroNumQueuedEnvelopes() {
StringBuilder str = new StringBuilder();
- str.append(String.format("==== %d outgoing connections ===\n", this.outConnections.size()));
+ str.append(String.format("==== %d outgoing connections ===\n", outConnections.size()));
- for (Map.Entry<RemoteReceiver, Object> entry : this.outConnections.entrySet()) {
+ for (Map.Entry<RemoteReceiver, Object> entry : outConnections.entrySet()) {
RemoteReceiver receiver = entry.getKey();
Object value = entry.getValue();
@@ -294,41 +316,52 @@ public class NettyConnectionManager implements NetworkConnectionManager {
private final RemoteReceiver receiver;
- private ChannelInBuildup(Bootstrap out, RemoteReceiver receiver) {
+ private final NetworkConnectionManager connectionManager;
+
+ private final int closeAfterIdleMs;
+
+ private ChannelInBuildup(
+ Bootstrap out,
+ RemoteReceiver receiver,
+ NetworkConnectionManager connectionManager,
+ int closeAfterIdleMs) {
+
this.out = out;
this.receiver = receiver;
+ this.connectionManager = connectionManager;
+ this.closeAfterIdleMs = closeAfterIdleMs;
}
private void handInChannel(OutboundConnectionQueue c) {
- synchronized (this.lock) {
- this.channel = c;
- this.lock.notifyAll();
+ synchronized (lock) {
+ channel = c;
+ lock.notifyAll();
}
}
- private void notifyOfError(Throwable error) {
- synchronized (this.lock) {
- this.error = error;
- this.lock.notifyAll();
+ private void notifyOfError(Throwable t) {
+ synchronized (lock) {
+ error = t;
+ lock.notifyAll();
}
}
private OutboundConnectionQueue waitForChannel() throws IOException {
- synchronized (this.lock) {
- while (this.error == null && this.channel == null) {
+ synchronized (lock) {
+ while (error == null && channel == null) {
try {
- this.lock.wait(2000);
+ lock.wait(2000);
} catch (InterruptedException e) {
throw new RuntimeException("Channel buildup interrupted.");
}
}
}
- if (this.error != null) {
+ if (error != null) {
throw new IOException("Connecting the channel failed: " + error.getMessage(), error);
}
- return this.channel;
+ return channel;
}
@Override
@@ -338,13 +371,14 @@ public class NettyConnectionManager implements NetworkConnectionManager {
LOG.debug(String.format("Channel %s connected", future.channel()));
}
- handInChannel(new OutboundConnectionQueue(future.channel()));
+ handInChannel(new OutboundConnectionQueue(
+ future.channel(), receiver, connectionManager, closeAfterIdleMs));
}
- else if (this.numRetries > 0) {
- LOG.debug(String.format("Connection request did not succeed, retrying (%d attempts left)", this.numRetries));
+ else if (numRetries > 0) {
+ LOG.debug(String.format("Connection request did not succeed, retrying (%d attempts left)", numRetries));
- this.out.connect(this.receiver.getConnectionAddress()).addListener(this);
- this.numRetries--;
+ out.connect(receiver.getConnectionAddress()).addListener(this);
+ numRetries--;
}
else {
if (future.getClass() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueue.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueue.java
index 621a259..eb76be5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueue.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueue.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.io.network.netty;
import io.netty.channel.Channel;
@@ -24,74 +23,137 @@ import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
-
+import io.netty.handler.timeout.IdleStateEvent;
+import io.netty.handler.timeout.IdleStateHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.runtime.io.network.Envelope;
+import org.apache.flink.runtime.io.network.NetworkConnectionManager;
+import org.apache.flink.runtime.io.network.RemoteReceiver;
import java.util.ArrayDeque;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
-public class OutboundConnectionQueue extends ChannelInboundHandlerAdapter implements ChannelFutureListener {
+public class OutboundConnectionQueue extends ChannelInboundHandlerAdapter {
+
+ private static enum QueueEvent {
+ TRIGGER_WRITE
+ }
private static final Log LOG = LogFactory.getLog(OutboundConnectionQueue.class);
+ private final ChannelWriteListener writeListener = new ChannelWriteListener();
+
private final Channel channel;
- private final ArrayDeque<Envelope> queuedEnvelopes = new ArrayDeque<Envelope>();
+ private final Queue<Envelope> queuedEnvelopes = new ArrayDeque<Envelope>();
- private final AtomicInteger numQueuedEnvelopes = new AtomicInteger(0);
+ private final RemoteReceiver receiver;
+
+ private final NetworkConnectionManager connectionManager;
+
+ // Flag to indicate whether a channel close was requested. This flag is true,
+ // iff there are no queued envelopes, when the channel is idling. After a
+ // successful close request, enqueue should return false.
+ private boolean hasRequestedClose = false;
+
+ public OutboundConnectionQueue(
+ Channel channel,
+ RemoteReceiver receiver,
+ NetworkConnectionManager connectionManager,
+ int closeAfterIdleForMs) {
- public OutboundConnectionQueue(Channel channel) {
this.channel = channel;
+ this.receiver = receiver;
+ this.connectionManager = connectionManager;
- channel.pipeline().addFirst(this);
+ channel.pipeline().addFirst("Outbound Connection Queue", this);
+ channel.pipeline().addFirst("Idle State Handler",
+ new IdleStateHandler(0, 0, closeAfterIdleForMs, TimeUnit.MILLISECONDS));
}
/**
- * Enqueues an envelope to be sent later.
+ * Enqueues an envelope to be sent.
* <p/>
* This method is always invoked by the task thread that wants the envelope sent.
+ * <p/>
+ * If this method returns <code>false</code>, the channel cannot be used to enqueue
+ * envelopes any more and the caller needs to establish a new connection to the target.
+ * The current envelope needs to be enqueued at the new channel.
*
- * @param env The envelope to be sent.
+ * @param env the envelope to be sent
+ * @return true, if successfully enqueued or false, if the channel was requested to be closed
*/
- public void enqueue(Envelope env) {
- // the user event trigger ensure thread-safe hand-over of the envelope
- this.channel.pipeline().fireUserEventTriggered(env);
- }
+ public boolean enqueue(Envelope env) {
+ boolean triggerWrite;
- @Override
- public void userEventTriggered(ChannelHandlerContext ctx, Object envelopeToEnqueue) throws Exception {
- boolean triggerWrite = this.queuedEnvelopes.isEmpty();
+ synchronized (channel) {
+ if (hasRequestedClose) {
+ // The caller has to ensure that the envelope gets queued to
+ // a new channel.
+ return false;
+ }
+
+ // Initiate envelope processing, after the queue state has
+ // changed from empty to non-empty.
+ triggerWrite = queuedEnvelopes.isEmpty();
- this.queuedEnvelopes.addLast((Envelope) envelopeToEnqueue);
- this.numQueuedEnvelopes.incrementAndGet();
+ queuedEnvelopes.add(env);
+ }
if (triggerWrite) {
- writeAndFlushNextEnvelopeIfPossible();
+ channel.pipeline().fireUserEventTriggered(QueueEvent.TRIGGER_WRITE);
}
- }
- @Override
- public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
- writeAndFlushNextEnvelopeIfPossible();
+ return true;
}
@Override
- public void operationComplete(ChannelFuture future) throws Exception {
- if (future.isSuccess()) {
+ public void userEventTriggered(ChannelHandlerContext ctx, Object event) throws Exception {
+ if (event.getClass() == QueueEvent.class) {
+ // Initiate envelope processing, after the queue state has
+ // changed from empty to non-empty.
writeAndFlushNextEnvelopeIfPossible();
}
- else if (future.cause() != null) {
- exceptionOccurred(future.cause());
+ else if (event.getClass() == IdleStateEvent.class) {
+ // Channel idle => try to close
+ boolean closeConnection = false;
+
+ // Only close the connection, if there are no queued envelopes. We have
+ // to ensure that there is no race between closing the channel and
+ // enqueuing a new envelope.
+ synchronized (channel) {
+ if (queuedEnvelopes.isEmpty() && !hasRequestedClose) {
+
+ hasRequestedClose = true;
+
+ closeConnection = true;
+
+ // Notify the connection manager that this channel has been
+ // closed.
+ connectionManager.close(receiver);
+ }
+ }
+
+ if (closeConnection) {
+ ctx.close().addListener(new ChannelCloseListener());
+ }
}
else {
- exceptionOccurred(new Exception("Envelope send aborted."));
+ throw new IllegalStateException("Triggered unknown event.");
}
}
+ @Override
+ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
+ writeAndFlushNextEnvelopeIfPossible();
+ }
+
public int getNumQueuedEnvelopes() {
- return this.numQueuedEnvelopes.intValue();
+ synchronized (channel) {
+ return queuedEnvelopes.size();
+ }
}
@Override
@@ -100,16 +162,48 @@ public class OutboundConnectionQueue extends ChannelInboundHandlerAdapter implem
}
private void writeAndFlushNextEnvelopeIfPossible() {
- if (this.channel.isWritable() && !this.queuedEnvelopes.isEmpty()) {
- Envelope nextEnvelope = this.queuedEnvelopes.pollFirst();
- this.numQueuedEnvelopes.decrementAndGet();
+ Envelope nextEnvelope = null;
+
+ synchronized (channel) {
+ if (channel.isWritable() && !queuedEnvelopes.isEmpty()) {
+ nextEnvelope = queuedEnvelopes.poll();
+ }
+ }
+
+ if (nextEnvelope != null) {
+ channel.writeAndFlush(nextEnvelope).addListener(writeListener);
+ }
+ }
+
+ private class ChannelWriteListener implements ChannelFutureListener {
+
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (future.isSuccess()) {
+ writeAndFlushNextEnvelopeIfPossible();
+ }
+ else {
+ exceptionOccurred(future.cause() == null
+ ? new Exception("Envelope send aborted.")
+ : future.cause());
+ }
+ }
+ }
+
+ private class ChannelCloseListener implements ChannelFutureListener {
- this.channel.writeAndFlush(nextEnvelope).addListener(this);
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (!future.isSuccess()) {
+ exceptionOccurred(future.cause() == null
+ ? new Exception("Close failed.")
+ : future.cause());
+ }
}
}
private void exceptionOccurred(Throwable t) throws Exception {
- LOG.error(String.format("An exception occurred in Channel %s: %s", this.channel, t.getMessage()));
+ LOG.error(String.format("Exception in Channel %s: %s", channel, t.getMessage()));
throw new Exception(t);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
index 69d4869..7521218 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.taskmanager;
import java.io.File;
@@ -318,17 +317,13 @@ public class TaskManager implements TaskOperationProtocol {
ConfigConstants.TASK_MANAGER_NET_NUM_OUT_THREADS_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS);
- int lowWaterMark = GlobalConfiguration.getInteger(
- ConfigConstants.TASK_MANAGER_NET_NETTY_LOW_WATER_MARK,
- ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK);
-
- int highWaterMark = GlobalConfiguration.getInteger(
- ConfigConstants.TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK,
- ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK);
+ int closeAfterIdleForMs = GlobalConfiguration.getInteger(
+ ConfigConstants.TASK_MANAGER_NET_CLOSE_AFTER_IDLE_FOR_MS_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_NET_CLOSE_AFTER_IDLE_FOR_MS);
networkConnectionManager = new NettyConnectionManager(
localInstanceConnectionInfo.address(), localInstanceConnectionInfo.dataPort(),
- bufferSize, numInThreads, numOutThreads, lowWaterMark, highWaterMark);
+ bufferSize, numInThreads, numOutThreads, closeAfterIdleForMs);
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
index dfb654d..cfb6ae5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyConnectionManagerTest.java
@@ -16,18 +16,16 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.io.network.netty;
-import org.junit.Assert;
-
import org.apache.flink.runtime.io.network.ChannelManager;
import org.apache.flink.runtime.io.network.Envelope;
import org.apache.flink.runtime.io.network.RemoteReceiver;
import org.apache.flink.runtime.io.network.channels.ChannelID;
-import org.apache.flink.runtime.io.network.netty.NettyConnectionManager;
import org.apache.flink.runtime.jobgraph.JobID;
+import org.junit.Assert;
import org.mockito.Matchers;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -38,158 +36,206 @@ import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
public class NettyConnectionManagerTest {
private final static long RANDOM_SEED = 520346508276087l;
- private final static Random random = new Random(RANDOM_SEED);
+ private Random rand = new Random(RANDOM_SEED);
- private final static int BIND_PORT = 20000;
+ private NettyConnectionManager senderManager;
- private final static int BUFFER_SIZE = 32 * 1024;
+ private NettyConnectionManager receiverManager;
- public void testEnqueueRaceAndDeadlockFreeMultipleChannels() throws Exception {
- Integer[][] configs = new Integer[][]{
- {64, 4096, 1, 1, 1},
- {128, 2048, 1, 1, 1},
- {256, 1024, 1, 1, 1},
- {512, 512, 1, 1, 1},
- {64, 4096, 4, 1, 1},
- {128, 2048, 4, 1, 1},
- {256, 1024, 4, 1, 1},
- {512, 512, 4, 1, 1},
- {64, 4096, 4, 2, 2},
- {128, 2048, 4, 2, 2},
- {256, 1024, 4, 2, 2},
- {512, 512, 4, 2, 2}
- };
+ private ChannelManager channelManager;
- for (Integer[] params : configs) {
- System.out.println(String.format("Running %s with config: %d sub tasks, %d envelopes to send per subtasks, "
- + "%d num channels, %d num in threads, %d num out threads.",
- "testEnqueueRaceAndDeadlockFreeMultipleChannels", params[0], params[1], params[2], params[3], params[4]));
+ private RemoteReceiver[] receivers;
- long start = System.currentTimeMillis();
- doTestEnqueueRaceAndDeadlockFreeMultipleChannels(params[0], params[1], params[2], params[3], params[4]);
- long end = System.currentTimeMillis();
+ private CountDownLatch receivedAllEnvelopesLatch;
- System.out.println(String.format("Runtime: %d ms.", (end - start)));
- }
- }
+ private void initTest(
+ int numProducers,
+ final int numEnvelopesPerProducer,
+ int numInThreads,
+ int numOutThreads,
+ int closeAfterIdleForMs) throws Exception {
- private void doTestEnqueueRaceAndDeadlockFreeMultipleChannels(
- int numSubtasks, final int numToSendPerSubtask, int numChannels, int numInThreads, int numOutThreads)
- throws Exception {
+ final InetAddress bindAddress = InetAddress.getLocalHost();
+ final int bindPort = 20000;
+ final int bufferSize = 32 * 1024;
- final InetAddress localhost = InetAddress.getLocalHost();
- final CountDownLatch latch = new CountDownLatch(numSubtasks);
+ senderManager = Mockito.spy(new NettyConnectionManager(
+ bindAddress, bindPort, bufferSize, numInThreads, numOutThreads, closeAfterIdleForMs));
- // --------------------------------------------------------------------
- // setup
- // --------------------------------------------------------------------
- ChannelManager channelManager = mock(ChannelManager.class);
- doAnswer(new VerifyEnvelopes(latch, numToSendPerSubtask))
- .when(channelManager).dispatchFromNetwork(Matchers.<Envelope>anyObject());
+ receiverManager = new NettyConnectionManager(
+ bindAddress, bindPort + 1, bufferSize, numInThreads, numOutThreads, closeAfterIdleForMs);
- final NettyConnectionManager senderConnManager = new NettyConnectionManager(localhost, BIND_PORT, BUFFER_SIZE,
- numInThreads, numOutThreads, -1, -1);
- senderConnManager.start(channelManager);
+ channelManager = Mockito.mock(ChannelManager.class);
- NettyConnectionManager receiverConnManager = new NettyConnectionManager(localhost, BIND_PORT + 1, BUFFER_SIZE,
- numInThreads, numOutThreads, -1, -1);
- receiverConnManager.start(channelManager);
+ senderManager.start(channelManager);
+ receiverManager.start(channelManager);
+
+ receivers = new RemoteReceiver[numProducers];
+ for (int i = 0; i < numProducers; i++) {
+ receivers[i] = new RemoteReceiver(new InetSocketAddress(bindPort + 1), i);
+ }
// --------------------------------------------------------------------
- // start sender threads
- // --------------------------------------------------------------------
- RemoteReceiver[] receivers = new RemoteReceiver[numChannels];
- for (int i = 0; i < numChannels; i++) {
- receivers[i] = new RemoteReceiver(new InetSocketAddress(localhost, BIND_PORT + 1), i);
- }
+ receivedAllEnvelopesLatch = new CountDownLatch(numProducers);
- for (int i = 0; i < numSubtasks; i++) {
- final RemoteReceiver receiver = receivers[random.nextInt(numChannels)];
+ final ConcurrentMap<ChannelID, Integer> receivedSequenceNums =
+ new ConcurrentHashMap<ChannelID, Integer>();
- final AtomicInteger seqNum = new AtomicInteger(0);
- final JobID jobId = new JobID();
- final ChannelID channelId = new ChannelID();
+ // Verifies that the sequence numbers of each producer are received
+ // in ascending incremental order. In addition, manages a latch to
+ // allow synchronization after all envelopes have been received.
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ final Envelope env = (Envelope) invocation.getArguments()[0];
- new Thread(new Runnable() {
- @Override
- public void run() {
- // enqueue envelopes with ascending seq numbers
- while (seqNum.get() < numToSendPerSubtask) {
- try {
- Envelope env = new Envelope(seqNum.getAndIncrement(), jobId, channelId);
- senderConnManager.enqueue(env, receiver);
- } catch (IOException e) {
- throw new RuntimeException("Unexpected exception while enqueuing envelope.");
- }
- }
+ final int currentSeqNum = env.getSequenceNumber();
+ final ChannelID cid = env.getSource();
+
+ if (currentSeqNum < 0 || currentSeqNum >= numEnvelopesPerProducer) {
+ Assert.fail("Received more envelopes than expected from " + cid);
+ }
+
+ Integer previousSeqNum = receivedSequenceNums.put(cid, currentSeqNum);
+
+ if (previousSeqNum != null) {
+ String errMsg = String.format("Received %s with unexpected sequence number.", env);
+ Assert.assertEquals(errMsg, previousSeqNum + 1, currentSeqNum);
}
- }).start();
- }
- latch.await();
+ if (currentSeqNum == numEnvelopesPerProducer - 1) {
+ receivedAllEnvelopesLatch.countDown();
+ }
- senderConnManager.shutdown();
- receiverConnManager.shutdown();
+ return null;
+ }
+ }).when(channelManager).dispatchFromNetwork(Matchers.any(Envelope.class));
}
- /**
- * Verifies correct ordering of received envelopes (per envelope source channel ID).
- */
- private class VerifyEnvelopes implements Answer<Void> {
+ private void finishTest() throws IOException {
+ senderManager.shutdown();
+ receiverManager.shutdown();
+ }
- private final ConcurrentMap<ChannelID, Integer> received = new ConcurrentHashMap<ChannelID, Integer>();
+ // ------------------------------------------------------------------------
+
+ // Verifies that there are no race conditions or dead locks with
+ // concurrent enqueues and closes.
+ public void testConcurrentEnqueueAndClose() throws Exception {
+ Integer[][] configs = new Integer[][]{
+ // No close after idle
+ {64, 4096, 1, 1, 1, 0, 0},
+ {128, 2048, 1, 1, 1, 0, 0},
+ {256, 1024, 1, 1, 1, 0, 0},
+ {512, 512, 1, 1, 1, 0, 0},
+
+ {64, 4096, 4, 1, 1, 0, 0},
+ {128, 2048, 4, 1, 1, 0, 0},
+ {256, 1024, 4, 1, 1, 0, 0},
+ {512, 512, 4, 1, 1, 0, 0},
+
+ {64, 4096, 4, 2, 2, 0, 0},
+ {128, 2048, 4, 2, 2, 0, 0},
+ {256, 1024, 4, 2, 2, 0, 0},
+ {512, 512, 4, 2, 2, 0, 0},
+ // Note: these need plenty of heap space for the threads
+ {1024, 256, 4, 2, 2, 0, 0},
+ {2048, 128, 4, 2, 2, 0, 0},
+ {4096, 64, 4, 2, 2, 0, 0},
+
+ // With close after idle
+ {4, 1024, 1, 1, 1, 40, 80},
+ {8, 1024, 1, 1, 1, 40, 80},
+ {16, 1024, 1, 1, 1, 40, 80},
+ {32, 1024, 1, 1, 1, 40, 80},
+ {64, 1024, 1, 1, 1, 40, 80},
+
+ {16, 1024, 4, 1, 1, 40, 80},
+ {32, 1024, 4, 1, 1, 40, 80},
+ {64, 1024, 4, 1, 1, 40, 80},
+
+ {16, 1024, 4, 2, 2, 40, 80},
+ {32, 1024, 4, 2, 2, 40, 80},
+ {64, 1024, 4, 2, 2, 40, 80}
+ };
- private final CountDownLatch latch;
+ for (Integer[] params : configs) {
+ System.out.println(String.format("Running testConcurrentEnqueueAndClose with config: " +
+ "%d producers, %d envelopes per producer, %d num channels, " +
+ "%d num in threads, %d num out threads, " +
+ "%d ms min sleep time, %d ms max sleep time.",
+ params[0], params[1], params[2], params[3], params[4], params[5], params[6]));
- private final int numExpectedEnvelopesPerSubtask;
+ long start = System.currentTimeMillis();
+ doTestConcurrentEnqueueAndClose(params[0], params[1], params[2], params[3], params[4], params[5], params[6]);
+ long end = System.currentTimeMillis();
- private VerifyEnvelopes(CountDownLatch latch, int numExpectedEnvelopesPerSubtask) {
- this.latch = latch;
- this.numExpectedEnvelopesPerSubtask = numExpectedEnvelopesPerSubtask;
+ System.out.println(String.format("Runtime: %d ms.", (end - start)));
}
+ }
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Envelope env = (Envelope) invocation.getArguments()[0];
+ private void doTestConcurrentEnqueueAndClose(
+ int numProducers,
+ final int numEnvelopesPerProducer,
+ final int numChannels,
+ int numInThreads,
+ int numOutThreads,
+ final int minSleepTimeMs,
+ final int maxSleepTimeMs) throws Exception {
+
+ // The idle time before a close is requested is 1/4th of the min sleep
+ // time of each producer. Depending on the number of concurrent producers
+ // and number of envelopes to send per producer, this will result in a
+ // variable number of close requests.
+ initTest(numProducers, numEnvelopesPerProducer, numInThreads, numOutThreads, minSleepTimeMs / 4);
- ChannelID channelId = env.getSource();
- int seqNum = env.getSequenceNumber();
+ // --------------------------------------------------------------------
- if (seqNum == 0) {
- Integer previousSeqNum = this.received.putIfAbsent(channelId, seqNum);
+ Runnable[] producers = new Runnable[numProducers];
- String msg = String.format("Received envelope from %s before, but current seq num is 0", channelId);
- Assert.assertNull(msg, previousSeqNum);
- }
- else {
- boolean isExpectedPreviousSeqNum = this.received.replace(channelId, seqNum - 1, seqNum);
+ for (int i = 0; i < numProducers; i++) {
+ producers[i] = new Runnable() {
+ @Override
+ public void run() {
+ final JobID jid = new JobID();
+ final ChannelID cid = new ChannelID();
+ final RemoteReceiver receiver = receivers[rand.nextInt(numChannels)];
- String msg = String.format("Received seq num %d from %s, but previous was not %d.",
- seqNum, channelId, seqNum - 1);
- Assert.assertTrue(msg, isExpectedPreviousSeqNum);
- }
+ for (int sequenceNum = 0; sequenceNum < numEnvelopesPerProducer; sequenceNum++) {
+ try {
+ senderManager.enqueue(new Envelope(sequenceNum, jid, cid), receiver);
- // count down the latch if all envelopes received for this source
- if (seqNum == numExpectedEnvelopesPerSubtask - 1) {
- this.latch.countDown();
- }
+ int sleepTime = rand.nextInt((maxSleepTimeMs - minSleepTimeMs) + 1) + minSleepTimeMs;
+ Thread.sleep(sleepTime);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+ }
- return null;
+ for (int i = 0; i < numProducers; i++) {
+ new Thread(producers[i], "Producer " + i).start();
}
+
+ // --------------------------------------------------------------------
+
+ while (receivedAllEnvelopesLatch.getCount() != 0) {
+ receivedAllEnvelopesLatch.await();
+ }
+
+ finishTest();
}
private void runAllTests() throws Exception {
- testEnqueueRaceAndDeadlockFreeMultipleChannels();
+ testConcurrentEnqueueAndClose();
System.out.println("Done.");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/52512636/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
new file mode 100644
index 0000000..1f4d98a
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundConnectionQueueTest.java
@@ -0,0 +1,606 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.netty;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.embedded.EmbeddedChannel;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.timeout.IdleStateEvent;
+import org.apache.flink.runtime.io.network.Envelope;
+import org.apache.flink.runtime.io.network.NetworkConnectionManager;
+import org.apache.flink.runtime.io.network.RemoteReceiver;
+import org.apache.flink.runtime.io.network.channels.ChannelID;
+import org.apache.flink.runtime.jobgraph.JobID;
+import org.junit.Assert;
+import org.mockito.Mockito;
+import org.powermock.reflect.Whitebox;
+
+import java.net.InetAddress;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+
+public class OutboundConnectionQueueTest {
+
+ private final static long RANDOM_SEED = 520346508276087l;
+
+ private final Object lock = new Object();
+
+ private Channel channel;
+
+ private NetworkConnectionManager connectionManager;
+
+ private RemoteReceiver receiver;
+
+ private OutboundConnectionQueue queue;
+
+ private TestControlHandler controller;
+
+ private TestVerificationHandler verifier;
+
+ private Throwable exception;
+
+ private void initTest(boolean autoTriggerWrite) {
+ controller = new TestControlHandler(autoTriggerWrite);
+ verifier = new TestVerificationHandler();
+
+ channel = Mockito.spy(new EmbeddedChannel(new ChannelInboundHandlerAdapter() {
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ exception = cause;
+ super.exceptionCaught(ctx, cause);
+ }
+ }));
+
+ connectionManager = Mockito.mock(NetworkConnectionManager.class);
+
+ receiver = Mockito.mock(RemoteReceiver.class);
+
+ queue = new OutboundConnectionQueue(channel, receiver, connectionManager, 0);
+
+ channel.pipeline().addFirst("Test Control Handler", controller);
+ channel.pipeline().addFirst("Test Verification Handler", verifier);
+
+ exception = null;
+
+ // The testing pipeline looks as follows:
+ // - Test Verification Handler [OUT]
+ // - Test Control Handler [IN]
+ // - Idle State Handler [IN/OUT] [added by OutboundConnectionQueue]
+ // - Outbound queue (SUT) [IN] [added by OutboundConnectionQueue]
+ // - Exception setter [IN] [EmbeddedChannel constructor]
+ }
+
+ /**
+ * Verifies that the channel is closed after an idle event, when
+ * there are no queued envelopes.
+ */
+ public void testClose() throws Exception {
+ initTest(false);
+
+ JobID jid = new JobID();
+ ChannelID cid = new ChannelID();
+
+ Assert.assertTrue(queue.enqueue(new Envelope(1, jid, cid)));
+ Assert.assertTrue(queue.enqueue(new Envelope(2, jid, cid)));
+ Assert.assertTrue(queue.enqueue(new Envelope(3, jid, cid)));
+
+ controller.triggerWrite();
+
+ controller.fireIdle();
+
+ verifier.waitForClose();
+
+ verifier.verifyEnvelopeReceived(cid, 3);
+
+ Mockito.verify(connectionManager, Mockito.times(1)).close(Mockito.any(RemoteReceiver.class));
+ }
+
+ /**
+ * Verifies that the channel is not closed while there are queued
+ * envelopes.
+ */
+ public void testCloseWithQueuedEnvelopes() throws Exception {
+ initTest(true);
+
+ final JobID jid = new JobID();
+ final ChannelID cid = new ChannelID();
+ final CountDownLatch sync = verifier.waitForEnvelopes(3, cid);
+
+ // Make channel not writable => envelopes are queued
+ Mockito.when(channel.isWritable()).thenReturn(false);
+
+ Assert.assertTrue(queue.enqueue(new Envelope(1, jid, cid)));
+ Assert.assertTrue(queue.enqueue(new Envelope(2, jid, cid)));
+ Assert.assertTrue(queue.enqueue(new Envelope(3, jid, cid)));
+
+ // Verify idle event doesn't close channel
+ controller.fireIdle();
+
+ Mockito.verify(connectionManager, Mockito.times(0)).close(Mockito.any(RemoteReceiver.class));
+
+ boolean hasRequestedClose = Whitebox.getInternalState(queue, "hasRequestedClose");
+ Assert.assertFalse("Close request while envelope in flight.", hasRequestedClose);
+
+ // Change writability of channel back to writable
+ Mockito.when(channel.isWritable()).thenReturn(true);
+ channel.pipeline().fireChannelWritabilityChanged();
+
+ // Wait for the processing of queued envelopes
+ while (sync.getCount() != 0) {
+ sync.await();
+ }
+
+ verifier.verifyEnvelopeReceived(cid, 3);
+
+ // Now close again
+ controller.fireIdle();
+ verifier.waitForClose();
+
+ Mockito.verify(connectionManager, Mockito.times(1)).close(Mockito.any(RemoteReceiver.class));
+ }
+
+ /**
+ * Verifies that envelopes are delegated back to the connection
+ * manager after a close.
+ */
+ public void testEnqueueAfterClose() throws Exception {
+ initTest(true);
+
+ // Immediately close the channel
+ controller.fireIdle();
+ verifier.waitForClose();
+
+ Assert.assertFalse(queue.enqueue(new Envelope(1, new JobID(), new ChannelID())));
+ }
+
+ /**
+ * Verifies that multiple idle events are handled correctly.
+ */
+ public void testMultipleIdleEvents() throws Exception {
+ initTest(true);
+
+ controller.fireIdle();
+ verifier.waitForClose();
+
+ controller.fireIdle();
+
+ // Second close should not cause an exception in the
+ // verification handler.
+ Assert.assertNull(exception);
+ }
+
+ /**
+ * Verifies that unknown user events throw an exception.
+ */
+ public void testUnknownUserEvent() throws Exception {
+ initTest(true);
+
+ Assert.assertNull(exception);
+
+ controller.context.fireUserEventTriggered("Unknown user event");
+
+ Assert.assertNotNull(exception);
+ Assert.assertTrue(exception instanceof IllegalStateException);
+ }
+
+ // ------------------------------------------------------------------------
+
+ public void testConcurrentEnqueueAndClose() throws Exception {
+ Integer[][] configs = new Integer[][]{
+ {1, 512, 0, 0},
+ {1, 512, 40, 80},
+ {2, 512, 40, 80},
+ {4, 512, 40, 80},
+ {8, 512, 40, 80},
+ {32, 512, 40, 80},
+ {128, 512, 40, 80},
+ {256, 512, 40, 80},
+ {512, 512, 40, 80}
+ };
+
+ for (Integer[] params : configs) {
+ System.out.println(String.format(
+ "Running %s with config: %d producers, %d envelopes to send per producer, " +
+ "%d ms min sleep time, %d ms max sleep time.", "testConcurrentEnqueueAndClose",
+ params[0], params[1], params[2], params[3]));
+
+ long start = System.currentTimeMillis();
+ doTestConcurrentEnqueueAndClose(params[0], params[1], params[2], params[3]);
+ long end = System.currentTimeMillis();
+
+ System.out.println(String.format("Runtime: %d ms.", (end - start)));
+ }
+ }
+
+ /**
+ * Verifies that concurrent enqueue and close events are handled
+ * correctly.
+ */
+ private void doTestConcurrentEnqueueAndClose(
+ final int numProducers,
+ final int numEnvelopesPerProducer,
+ final int minSleepTimeMs,
+ final int maxSleepTimeMs) throws Exception {
+
+ final InetAddress bindHost = InetAddress.getLocalHost();
+ final int bindPort = 20000;
+
+ // Testing concurrent enqueue and close requires real TCP channels,
+ // because Netty's testing EmbeddedChannel does not implement the
+ // same threading model as the NioEventLoopGroup (for example there
+ // is no difference between being IN and OUTSIDE of the event loop
+ // thread).
+
+ final ServerBootstrap in = new ServerBootstrap();
+ in.group(new NioEventLoopGroup(1))
+ .channel(NioServerSocketChannel.class)
+ .localAddress(bindHost, bindPort)
+ .childHandler(new ChannelInitializer<SocketChannel>() {
+ @Override
+ public void initChannel(SocketChannel channel) throws Exception {
+ channel.pipeline()
+ .addLast(new ChannelInboundHandlerAdapter());
+ }
+ });
+
+ final Bootstrap out = new Bootstrap();
+ out.group(new NioEventLoopGroup(1))
+ .channel(NioSocketChannel.class)
+ .handler(new ChannelInitializer<SocketChannel>() {
+ @Override
+ public void initChannel(SocketChannel channel) throws Exception {
+ channel.pipeline()
+ .addLast(new ChannelOutboundHandlerAdapter());
+ }
+ })
+ .option(ChannelOption.TCP_NODELAY, false)
+ .option(ChannelOption.SO_KEEPALIVE, true);
+
+ in.bind().sync();
+
+ // --------------------------------------------------------------------
+
+ // The testing pipeline looks as follows:
+ // - Test Verification Handler [OUT]
+ // - Test Control Handler [IN]
+ // - Idle State Handler [IN/OUT] [added by OutboundConnectionQueue]
+ // - Outbound queue (SUT) [IN] [added by OutboundConnectionQueue]
+
+ channel = out.connect(bindHost, bindPort).sync().channel();
+
+ queue = new OutboundConnectionQueue(channel, receiver, connectionManager, 0);
+
+ controller = new TestControlHandler(true);
+ verifier = new TestVerificationHandler();
+
+ channel.pipeline().addFirst("Test Control Handler", controller);
+ channel.pipeline().addFirst("Test Verification Handler", verifier);
+
+ // --------------------------------------------------------------------
+
+ final Random rand = new Random(RANDOM_SEED);
+
+ // Every producer works on their local reference of the queue and only
+ // updates it to the new channel when enqueue returns false, which
+ // should only happen if the channel has been closed.
+ final ConcurrentMap<ChannelID, OutboundConnectionQueue> producerQueues =
+ new ConcurrentHashMap<ChannelID, OutboundConnectionQueue>();
+
+ final ChannelID[] ids = new ChannelID[numProducers];
+
+ for (int i = 0; i < numProducers; i++) {
+ ids[i] = new ChannelID();
+
+ producerQueues.put(ids[i], queue);
+ }
+
+ final CountDownLatch receivedAllEnvelopesLatch =
+ verifier.waitForEnvelopes(numEnvelopesPerProducer - 1, ids);
+
+ final List<Channel> closedChannels = new ArrayList<Channel>();
+
+ // --------------------------------------------------------------------
+
+ final Runnable closer = new Runnable() {
+ @Override
+ public void run() {
+ while (receivedAllEnvelopesLatch.getCount() != 0) {
+ try {
+ controller.fireIdle();
+
+ // Test two idle events arriving "closely"
+ // after each other
+ if (rand.nextBoolean()) {
+ controller.fireIdle();
+ }
+
+ Thread.sleep(minSleepTimeMs / 2);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ };
+
+ final Runnable[] producers = new Runnable[numProducers];
+
+ for (int i = 0; i < numProducers; i++) {
+ final int index = i;
+
+ producers[i] = new Runnable() {
+ @Override
+ public void run() {
+ final JobID jid = new JobID();
+ final ChannelID cid = ids[index];
+
+ for (int j = 0; j < numEnvelopesPerProducer; j++) {
+ OutboundConnectionQueue localQueue = producerQueues.get(cid);
+
+ try {
+ // This code path is handled by the NetworkConnectionManager
+ // in production to enqueue the envelope either to the current
+ // channel or a new one if it was closed.
+ while (!localQueue.enqueue(new Envelope(j, jid, cid))) {
+ synchronized (lock) {
+ if (localQueue == queue) {
+ closedChannels.add(channel);
+
+ channel = out.connect(bindHost, bindPort).sync().channel();
+
+ queue = new OutboundConnectionQueue(channel, receiver, connectionManager, 0);
+
+ channel.pipeline().addFirst("Test Control Handler", controller);
+ channel.pipeline().addFirst("Test Verification Handler", verifier);
+ }
+ }
+
+ producerQueues.put(cid, queue);
+ localQueue = queue;
+ }
+
+ int sleepTime = rand.nextInt((maxSleepTimeMs - minSleepTimeMs) + 1) + minSleepTimeMs;
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+ }
+
+ for (int i = 0; i < numProducers; i++) {
+ new Thread(producers[i], "Producer " + i).start();
+ }
+
+ new Thread(closer, "Closer").start();
+
+ // --------------------------------------------------------------------
+
+ while (receivedAllEnvelopesLatch.getCount() != 0) {
+ receivedAllEnvelopesLatch.await();
+ }
+
+ // Final close, if the last close didn't make it.
+ synchronized (lock) {
+ if (channel != null) {
+ controller.fireIdle();
+ }
+ }
+
+ verifier.waitForClose();
+
+ // If the producers do not sleep after each envelope, the close
+ // should not make it through and no channel should have been
+ // added to the list of closed channels
+ if (minSleepTimeMs == 0 && maxSleepTimeMs == 0) {
+ Assert.assertEquals(0, closedChannels.size());
+ }
+
+ for (Channel ch : closedChannels) {
+ Assert.assertFalse(ch.isOpen());
+ }
+
+ System.out.println(closedChannels.size() + " channels were closed during execution.");
+
+ out.group().shutdownGracefully().sync();
+ in.group().shutdownGracefully().sync();
+ }
+
+ // ------------------------------------------------------------------------
+
+ // Handler to control the flow of Netty's custom user events. Used
+ // to fire idle events and manually trigger write events.
+ @ChannelHandler.Sharable
+ private static class TestControlHandler extends ChannelInboundHandlerAdapter {
+
+ private final Queue<Object> events = new ArrayDeque<Object>();
+
+ private final boolean forwardUserEvents;
+
+ private ChannelHandlerContext context;
+
+ private TestControlHandler(boolean forwardUserEvents) {
+ this.forwardUserEvents = forwardUserEvents;
+ }
+
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
+ synchronized (this) {
+ context = ctx;
+ }
+ }
+
+ @Override
+ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+ if (forwardUserEvents) {
+ ctx.fireUserEventTriggered(evt);
+ }
+ else {
+ events.add(evt);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+ super.exceptionCaught(ctx, cause);
+ }
+
+ public void triggerWrite() {
+ synchronized (this) {
+ if (!events.isEmpty()) {
+ context.fireUserEventTriggered(events.remove());
+ }
+ }
+ }
+
+ public void fireIdle() {
+ synchronized (this) {
+ context.fireUserEventTriggered(IdleStateEvent.ALL_IDLE_STATE_EVENT);
+ }
+ }
+ }
+
+ // Handler to verify writes and close events.
+ @ChannelHandler.Sharable
+ private static class TestVerificationHandler extends ChannelOutboundHandlerAdapter {
+
+ private final Map<ChannelID, Integer> receivedSequenceNums = new HashMap<ChannelID, Integer>();
+
+ private final Map<ChannelID, Integer> expectedSequenceNums = new HashMap<ChannelID, Integer>();
+
+ private final Map<ChannelID, CountDownLatch> envelopeLatches = new HashMap<ChannelID, CountDownLatch>();
+
+ private CountDownLatch closeLatch;
+
+ @Override
+ public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
+ closeLatch = new CountDownLatch(1);
+
+ super.handlerAdded(ctx);
+ }
+
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+ if (closeLatch.getCount() == 0) {
+ throw new IllegalStateException("Write on closed channel.");
+ }
+
+ if (msg.getClass() == Envelope.class) {
+ Envelope env = (Envelope) msg;
+
+ final int currentSeqNum = env.getSequenceNumber();
+ final ChannelID source = env.getSource();
+
+ Integer previousSeqNum = receivedSequenceNums.put(source, currentSeqNum);
+
+ if (previousSeqNum != null) {
+ String errMsg = String.format("Received %s with unexpected sequence number.", env);
+ Assert.assertEquals(errMsg, previousSeqNum + 1, currentSeqNum);
+ }
+
+ promise.setSuccess();
+
+ Integer expectedSeqNum = expectedSequenceNums.get(source);
+ if (expectedSeqNum != null && expectedSeqNum.equals(currentSeqNum)) {
+ envelopeLatches.remove(source).countDown();
+ }
+ }
+ }
+
+ @Override
+ public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception {
+ if (closeLatch.getCount() == 0) {
+ throw new IllegalStateException("Received multiple close events.");
+ }
+
+ super.close(ctx, promise);
+
+ closeLatch.countDown();
+ }
+
+ public void waitForClose() throws InterruptedException {
+ while (closeLatch.getCount() != 0) {
+ closeLatch.await();
+ }
+ }
+
+ /**
+ * Verifies that envelope with expected sequence number has been
+ * processed by the handler (or no envelope has been received by
+ * the given source, if expected sequence number is null).
+ */
+ public void verifyEnvelopeReceived(ChannelID source, Integer expectedSequenceNum) {
+ if (expectedSequenceNum == null && receivedSequenceNums.containsKey(source)) {
+ Assert.fail("Received unexpected envelope from channel " + source);
+ }
+ else if (receivedSequenceNums.containsKey(source)) {
+ Assert.assertEquals(expectedSequenceNum, receivedSequenceNums.get(source));
+ }
+ else {
+ Assert.fail("Did not receive any envelope from channel " + source);
+ }
+ }
+
+ public CountDownLatch waitForEnvelopes(Integer expectedSequenceNum, ChannelID... ids) {
+ CountDownLatch latch = new CountDownLatch(ids.length);
+
+ for (ChannelID id : ids) {
+ expectedSequenceNums.put(id, expectedSequenceNum);
+ envelopeLatches.put(id, latch);
+ }
+
+ return latch;
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ public void runAllTests() throws Exception {
+ testClose();
+ testCloseWithQueuedEnvelopes();
+ testEnqueueAfterClose();
+ testUnknownUserEvent();
+ testMultipleIdleEvents();
+ testConcurrentEnqueueAndClose();
+ }
+
+ public static void main(String[] args) throws Exception {
+ new OutboundConnectionQueueTest().runAllTests();
+ }
+}
[22/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
new file mode 100644
index 0000000..0c8bc97
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CrossFunction.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+
+
+/**
+ * @param <IN1> First input type
+ * @param <IN2> Second input type
+ * @param <OUT> Output type
+ */
+public interface CrossFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ /**
+ * User defined function for the cross operator.
+ *
+ * @param record1 Record from first input
+ * @param record2 Record from the second input
+ * @return result of cross UDF.
+ * @throws Exception
+ */
+ OUT cross(IN1 record1, IN2 record2) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
new file mode 100644
index 0000000..2f68477
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FilterFunction.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+import java.io.Serializable;
+
+public interface FilterFunction<T> extends Function, Serializable {
+
+ /**
+ * User defined function for a filter.
+ *
+ * @param value Incoming tuples
+ * @return true for tuples that are allowed to pass the filter
+ * @throws Exception
+ */
+ boolean filter(T value) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
new file mode 100644
index 0000000..b2c8f30
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.util.Collector;
+
+/**
+ * Generic interface used for combiners.
+ */
+public interface FlatCombineFunction<T> extends Function, Serializable {
+
+ void combine(Iterator<T> values, Collector<T> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
new file mode 100644
index 0000000..6a6b971
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatJoinFunction.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import org.apache.flink.util.Collector;
+
+import java.io.Serializable;
+
+
+public interface FlatJoinFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ void join (IN1 left, IN2 right, Collector<OUT> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
new file mode 100644
index 0000000..a8696cf
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatMapFunction.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import org.apache.flink.util.Collector;
+
+import java.io.Serializable;
+
+
+/**
+ *
+ * @param <T>
+ * @param <O>
+ */
+public interface FlatMapFunction<T, O> extends Function, Serializable {
+
+ /**
+ * The core method of FlatMappable. Takes an element from the input data set and transforms
+ * it into zero, one, or more elements.
+ *
+ * @param value The input value.
+ * @param out The collector for for emitting result values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ void flatMap(T value, Collector<O> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
index d3b7db4..c2a201f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
@@ -16,82 +16,13 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.functions;
-import org.apache.flink.configuration.Configuration;
-
/**
- * An base interface for all user-defined functions. This class defines methods for
- * the life cycle of the functions, as well as methods to access the context in which the functions
- * are executed.
+ * An base interface for all user-defined functions. This interface is empty in order
+ * to enable functions that are SAM (single abstract method) interfaces, so that they
+ * can be called as Java 8 lambdas
*/
public interface Function {
-
- /**
- * Initialization method for the function. It is called before the actual working methods
- * (like <i>map</i> or <i>join</i>) and thus suitable for one time setup work. For functions that
- * are part of an iteration, this method will be invoked at the beginning of each iteration superstep.
- * <p>
- * The configuration object passed to the function can be used for configuration and initialization.
- * The configuration contains all parameters that were configured on the function in the program
- * composition.
- *
- * <pre><blockquote>
- * public class MyMapper extends FilterFunction<String> {
- *
- * private String searchString;
- *
- * public void open(Configuration parameters) {
- * this.searchString = parameters.getString("foo");
- * }
- *
- * public boolean filter(String value) {
- * return value.equals(searchString);
- * }
- * }
- * </blockquote></pre>
- * <p>
- * By default, this method does nothing.
- *
- * @param parameters The configuration containing the parameters attached to the contract.
- *
- * @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
- * runtime catches an exception, it aborts the task and lets the fail-over logic
- * decide whether to retry the task execution.
- *
- * @see org.apache.flink.configuration.Configuration
- */
- void open(Configuration parameters) throws Exception;
- /**
- * Teardown method for the user code. It is called after the last call to the main working methods
- * (e.g. <i>map</i> or <i>join</i>). For functions that are part of an iteration, this method will
- * be invoked after each iteration superstep.
- * <p>
- * This method can be used for clean up work.
- *
- * @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
- * runtime catches an exception, it aborts the task and lets the fail-over logic
- * decide whether to retry the task execution.
- */
- void close() throws Exception;
-
-
- /**
- * Gets the context that contains information about the UDF's runtime.
- *
- * Context information are for example {@link org.apache.flink.api.common.accumulators.Accumulator}s
- * or the {@link org.apache.flink.api.common.cache.DistributedCache}.
- *
- * @return The UDF's runtime context.
- */
- RuntimeContext getRuntimeContext();
-
- /**
- * Sets the function's runtime context. Called by the framework when creating a parallel instance of the function.
- *
- * @param t The runtime context.
- */
- void setRuntimeContext(RuntimeContext t);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCoGrouper.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCoGrouper.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCoGrouper.java
deleted file mode 100644
index 59669a2..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCoGrouper.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import java.util.Iterator;
-
-import org.apache.flink.util.Collector;
-
-
-public interface GenericCoGrouper<V1, V2, O> extends Function {
-
- /**
- * This method must be implemented to provide a user implementation of a
- * coGroup. It is called for each two key-value pairs that share the same
- * key and come from different inputs.
- *
- * @param records1 The records from the first input which were paired with the key.
- * @param records2 The records from the second input which were paired with the key.
- * @param out A collector that collects all output pairs.
- */
- void coGroup(Iterator<V1> records1, Iterator<V2> records2, Collector<O> out) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
index ada4eeb..41cfa1d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCollectorMap.java
@@ -23,7 +23,7 @@ import org.apache.flink.util.Collector;
-public interface GenericCollectorMap<T, O> extends Function {
+public interface GenericCollectorMap<T, O> extends RichFunction {
void map(T record, Collector<O> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCombine.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCombine.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCombine.java
deleted file mode 100644
index 8dfe758..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCombine.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import java.util.Iterator;
-
-import org.apache.flink.util.Collector;
-
-/**
- * Generic interface used for combiners.
- */
-public interface GenericCombine<T> extends Function {
-
- void combine(Iterator<T> records, Collector<T> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCrosser.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCrosser.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCrosser.java
deleted file mode 100644
index 3de9b1d..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericCrosser.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import org.apache.flink.util.Collector;
-
-
-/**
- * @param <V1> First input type
- * @param <V2> Second input type
- * @param <O> Output type
- */
-public interface GenericCrosser<V1, V2, O> extends Function {
-
- /**
- * User defined function for the cross operator.
- *
- * @param record1 Record from first input
- * @param record2 Record from the second input
- * @param out Collector to submit resulting records.
- * @throws Exception
- */
- void cross(V1 record1, V2 record2, Collector<O> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFilter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFilter.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFilter.java
deleted file mode 100644
index f34b038..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFilter.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-
-public interface GenericFilter<T> extends Function {
-
- /**
- * User defined function for a filter.
- *
- * @param value Incoming tuples
- * @return true for tuples that are allowed to pass the filter
- * @throws Exception
- */
- boolean filter(T value) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFlatMap.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFlatMap.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFlatMap.java
deleted file mode 100644
index efb1d49..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericFlatMap.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import org.apache.flink.util.Collector;
-
-
-/**
- *
- * @param <T>
- * @param <O>
- */
-public interface GenericFlatMap<T, O> extends Function {
-
- /**
- * User defined function to perform transformations on records.
- * This method allows to submit an arbitrary number of records
- * per incoming tuple.
- *
- * @param record incoming record
- * @param out outgoing collector to return none, one or more records
- * @throws Exception
- */
- void flatMap(T record, Collector<O> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericGroupReduce.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericGroupReduce.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericGroupReduce.java
deleted file mode 100644
index e8d9910..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericGroupReduce.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import java.util.Iterator;
-
-import org.apache.flink.util.Collector;
-
-
-/**
- *
- * @param <T> Incoming types
- * @param <O> Outgoing types
- */
-public interface GenericGroupReduce<T, O> extends Function {
- /**
- *
- * The central function to be implemented for a reducer. The function receives per call one
- * key and all the values that belong to that key. Each key is guaranteed to be processed by exactly
- * one function call across all involved instances across all computing nodes.
- *
- * @param records All records that belong to the given input key.
- * @param out The collector to hand results to.
- * @throws Exception
- */
- void reduce(Iterator<T> records, Collector<O> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericJoiner.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericJoiner.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericJoiner.java
deleted file mode 100644
index 77c2ac9..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericJoiner.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-import org.apache.flink.util.Collector;
-
-
-public interface GenericJoiner<V1, V2, O> extends Function {
-
- void join(V1 value1, V2 value2, Collector<O> out) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericMap.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericMap.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericMap.java
deleted file mode 100644
index 316bf5d..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericMap.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-
-public interface GenericMap<T, O> extends Function {
-
- /**
- * A user-implemented function that modifies or transforms an incoming object and
- * returns the result.
- */
- O map(T record) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericReduce.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericReduce.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericReduce.java
deleted file mode 100644
index 9e75f2e..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GenericReduce.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.api.common.functions;
-
-
-public interface GenericReduce<T> extends Function {
-
- T reduce(T value1, T value2) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
new file mode 100644
index 0000000..984d1fd
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.flink.util.Collector;
+
+
+/**
+ *
+ * @param <T> Incoming types
+ * @param <O> Outgoing types
+ */
+public interface GroupReduceFunction<T, O> extends Function, Serializable {
+ /**
+ *
+ * The central function to be implemented for a reducer. The function receives per call one
+ * key and all the values that belong to that key. Each key is guaranteed to be processed by exactly
+ * one function call across all involved instances across all computing nodes.
+ *
+ * @param records All records that belong to the given input key.
+ * @param out The collector to hand results to.
+ * @throws Exception
+ */
+ void reduce(Iterator<T> values, Collector<O> out) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
new file mode 100644
index 0000000..02f526a
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/JoinFunction.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import java.io.Serializable;
+
+
+public interface JoinFunction<IN1, IN2, OUT> extends Function, Serializable {
+
+ OUT join(IN1 first, IN2 second) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
new file mode 100644
index 0000000..4e2520d
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/MapFunction.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+
+import java.io.Serializable;
+
+public interface MapFunction<T, O> extends Function, Serializable {
+
+ /**
+ * The core method of Mappable. Takes an element from the input data set and transforms
+ * it into exactly one element.
+ *
+ * @param value The input value.
+ * @returns The transformed value
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ O map(T value) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
new file mode 100644
index 0000000..04f690a
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/ReduceFunction.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+
+import java.io.Serializable;
+
+public interface ReduceFunction<T> extends Function, Serializable {
+
+ /**
+ * The core method of Reducible, combining two values into one value of the same type.
+ * The reduce function is consecutively applied to all values of a group until only a single value remains.
+ *
+ * @param value1 The first value to combine.
+ * @param value2 The second value to combine.
+ * @return The combined value of both input values.
+ *
+ * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
+ * to fail and may trigger recovery.
+ */
+ T reduce(T value1, T value2) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
new file mode 100644
index 0000000..ffc3ac2
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RichFunction.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.flink.api.common.functions;
+
+import org.apache.flink.configuration.Configuration;
+
+/**
+ * An base interface for all rich user-defined functions. This class defines methods for
+ * the life cycle of the functions, as well as methods to access the context in which the functions
+ * are executed.
+ */
+public interface RichFunction extends Function {
+
+ /**
+ * Initialization method for the function. It is called before the actual working methods
+ * (like <i>map</i> or <i>join</i>) and thus suitable for one time setup work. For functions that
+ * are part of an iteration, this method will be invoked at the beginning of each iteration superstep.
+ * <p>
+ * The configuration object passed to the function can be used for configuration and initialization.
+ * The configuration contains all parameters that were configured on the function in the program
+ * composition.
+ *
+ * <pre><blockquote>
+ * public class MyMapper extends FilterFunction<String> {
+ *
+ * private String searchString;
+ *
+ * public void open(Configuration parameters) {
+ * this.searchString = parameters.getString("foo");
+ * }
+ *
+ * public boolean filter(String value) {
+ * return value.equals(searchString);
+ * }
+ * }
+ * </blockquote></pre>
+ * <p>
+ * By default, this method does nothing.
+ *
+ * @param parameters The configuration containing the parameters attached to the contract.
+ *
+ * @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
+ * runtime catches an exception, it aborts the task and lets the fail-over logic
+ * decide whether to retry the task execution.
+ *
+ * @see org.apache.flink.configuration.Configuration
+ */
+ void open(Configuration parameters) throws Exception;
+
+ /**
+ * Teardown method for the user code. It is called after the last call to the main working methods
+ * (e.g. <i>map</i> or <i>join</i>). For functions that are part of an iteration, this method will
+ * be invoked after each iteration superstep.
+ * <p>
+ * This method can be used for clean up work.
+ *
+ * @throws Exception Implementations may forward exceptions, which are caught by the runtime. When the
+ * runtime catches an exception, it aborts the task and lets the fail-over logic
+ * decide whether to retry the task execution.
+ */
+ void close() throws Exception;
+
+
+ /**
+ * Gets the context that contains information about the UDF's runtime.
+ *
+ * Context information are for example {@link org.apache.flink.api.common.accumulators.Accumulator}s
+ * or the {@link org.apache.flink.api.common.cache.DistributedCache}.
+ *
+ * @return The UDF's runtime context.
+ */
+ RuntimeContext getRuntimeContext();
+
+ /**
+ * Sets the function's runtime context. Called by the framework when creating a parallel instance of the function.
+ *
+ * @param t The runtime context.
+ */
+ void setRuntimeContext(RuntimeContext t);
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index a5f0b19..3cf30ff 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -34,7 +34,7 @@ import org.apache.flink.api.common.cache.DistributedCache;
* the current degree of parallelism) and other constructs like accumulators and broadcast variables.
* <p>
* A function can, during runtime, obtain the RuntimeContext via a call to
- * {@link org.apache.flink.api.common.functions.AbstractFunction#getRuntimeContext()}.
+ * {@link AbstractRichFunction#getRuntimeContext()}.
*/
public interface RuntimeContext {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
new file mode 100644
index 0000000..bc4ffd0
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/FunctionUtils.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.functions.util;
+
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.configuration.Configuration;
+
+import java.lang.invoke.SerializedLambda;
+import java.lang.reflect.Method;
+
+public class FunctionUtils {
+
+
+ public static void openFunction (Function function, Configuration parameters) throws Exception{
+ if (function instanceof RichFunction) {
+ RichFunction richFunction = (RichFunction) function;
+ richFunction.open (parameters);
+ }
+ }
+
+ public static void closeFunction (Function function) throws Exception{
+ if (function instanceof RichFunction) {
+ RichFunction richFunction = (RichFunction) function;
+ richFunction.close ();
+ }
+ }
+
+ public static void setFunctionRuntimeContext (Function function, RuntimeContext context){
+ if (function instanceof RichFunction) {
+ RichFunction richFunction = (RichFunction) function;
+ richFunction.setRuntimeContext(context);
+ }
+ }
+
+ public static RuntimeContext getFunctionRuntimeContext (Function function, RuntimeContext defaultContext){
+ if (function instanceof RichFunction) {
+ RichFunction richFunction = (RichFunction) function;
+ return richFunction.getRuntimeContext();
+ }
+ else {
+ return defaultContext;
+ }
+ }
+
+ public static boolean isSerializedLambdaFunction(Function function) {
+ Class<?> clazz = function.getClass();
+ try {
+ Method replaceMethod = clazz.getDeclaredMethod("writeReplace");
+ replaceMethod.setAccessible(true);
+ Object serializedForm = replaceMethod.invoke(function);
+ if (serializedForm instanceof SerializedLambda) {
+ return true;
+ }
+ else {
+ return false;
+ }
+ }
+ catch (Exception e) {
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
index b140dda..c416765 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/Union.java
@@ -19,13 +19,13 @@
package org.apache.flink.api.common.operators;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
/**
* This operator represents a Union between two inputs.
*/
-public class Union<T> extends DualInputOperator<T, T, T, AbstractFunction> {
+public class Union<T> extends DualInputOperator<T, T, T, AbstractRichFunction> {
private final static String NAME = "Union";
@@ -34,7 +34,7 @@ public class Union<T> extends DualInputOperator<T, T, T, AbstractFunction> {
*/
public Union(BinaryOperatorInformation<T, T, T> operatorInfo) {
// we pass it an AbstractFunction, because currently all operators expect some form of UDF
- super(new UserCodeClassWrapper<AbstractFunction>(AbstractFunction.class), operatorInfo, NAME);
+ super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, NAME);
}
public Union(Operator<T> input1, Operator<T> input2) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
index ffcab50..66bea7f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/BulkIterationBase.java
@@ -29,7 +29,7 @@ import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.common.aggregators.AggregatorRegistry;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
import org.apache.flink.api.common.operators.IterationOperator;
import org.apache.flink.api.common.operators.Operator;
@@ -48,7 +48,7 @@ import org.apache.flink.util.Visitor;
/**
*
*/
-public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractFunction> implements IterationOperator {
+public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractRichFunction> implements IterationOperator {
private static String DEFAULT_NAME = "<Unnamed Bulk Iteration>";
@@ -78,7 +78,7 @@ public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractFunc
* @param name
*/
public BulkIterationBase(UnaryOperatorInformation<T, T> operatorInfo, String name) {
- super(new UserCodeClassWrapper<AbstractFunction>(AbstractFunction.class), operatorInfo, name);
+ super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name);
inputPlaceHolder = new PartialSolutionPlaceHolder<T>(this, this.getOperatorInfo());
}
@@ -230,7 +230,7 @@ public class BulkIterationBase<T> extends SingleInputOperator<T, T, AbstractFunc
/**
* Special Mapper that is added before a termination criterion and is only a container for an special aggregator
*/
- public static class TerminationCriterionMapper<X> extends AbstractFunction implements Serializable, GenericCollectorMap<X, Nothing> {
+ public static class TerminationCriterionMapper<X> extends AbstractRichFunction implements Serializable, GenericCollectorMap<X, Nothing> {
private static final long serialVersionUID = 1L;
private TerminationCriterionAggregator aggregator;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
index a9ae97c..4b85a31 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputOperator;
import org.apache.flink.api.common.operators.Ordering;
@@ -28,9 +28,9 @@ import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericCoGrouper
+ * @see org.apache.flink.api.common.functions.CoGroupFunction
*/
-public class CoGroupOperatorBase<IN1, IN2, OUT, FT extends GenericCoGrouper<IN1, IN2, OUT>> extends DualInputOperator<IN1, IN2, OUT, FT> {
+public class CoGroupOperatorBase<IN1, IN2, OUT, FT extends CoGroupFunction<IN1, IN2, OUT>> extends DualInputOperator<IN1, IN2, OUT, FT> {
/**
* The ordering for the order inside a group from input one.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CrossOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CrossOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CrossOperatorBase.java
index 33e150d..a66ea72 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CrossOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CrossOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericCrosser;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputOperator;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -28,9 +28,9 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericCrosser
+ * @see org.apache.flink.api.common.functions.CrossFunction
*/
-public class CrossOperatorBase<IN1, IN2, OUT, FT extends GenericCrosser<?, ?, ?>> extends DualInputOperator<IN1, IN2, OUT, FT> {
+public class CrossOperatorBase<IN1, IN2, OUT, FT extends CrossFunction<?, ?, ?>> extends DualInputOperator<IN1, IN2, OUT, FT> {
public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
super(udf, operatorInfo, name);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/DeltaIterationBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/DeltaIterationBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/DeltaIterationBase.java
index 89e5008..8e955b1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/DeltaIterationBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/DeltaIterationBase.java
@@ -23,7 +23,7 @@ import java.util.Collections;
import java.util.Map;
import org.apache.flink.api.common.aggregators.AggregatorRegistry;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputOperator;
import org.apache.flink.api.common.operators.IterationOperator;
@@ -51,7 +51,7 @@ import org.apache.flink.util.Visitor;
* This class is a subclass of {@code DualInputOperator}. The solution set is considered the first input, the
* workset is considered the second input.
*/
-public class DeltaIterationBase<ST, WT> extends DualInputOperator<ST, WT, ST, AbstractFunction> implements IterationOperator {
+public class DeltaIterationBase<ST, WT> extends DualInputOperator<ST, WT, ST, AbstractRichFunction> implements IterationOperator {
private final Operator<ST> solutionSetPlaceholder;
@@ -88,7 +88,7 @@ public class DeltaIterationBase<ST, WT> extends DualInputOperator<ST, WT, ST, Ab
}
public DeltaIterationBase(BinaryOperatorInformation<ST, WT, ST> operatorInfo, int[] keyPositions, String name) {
- super(new UserCodeClassWrapper<AbstractFunction>(AbstractFunction.class), operatorInfo, name);
+ super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name);
this.solutionSetKeyFields = keyPositions;
solutionSetPlaceholder = new SolutionSetPlaceHolder<ST>(this, new OperatorInformation<ST>(operatorInfo.getFirstInputType()));
worksetPlaceholder = new WorksetPlaceHolder<WT>(this, new OperatorInformation<WT>(operatorInfo.getSecondInputType()));
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
index 34896a2..3c28c43 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FilterOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -28,9 +28,9 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericFlatMap
+ * @see org.apache.flink.api.common.functions.FlatMapFunction
*/
-public class FilterOperatorBase<T, FT extends GenericFlatMap<T, T>> extends SingleInputOperator<T, T, FT> {
+public class FilterOperatorBase<T, FT extends FlatMapFunction<T, T>> extends SingleInputOperator<T, T, FT> {
public FilterOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<T, T> operatorInfo, String name) {
super(udf, operatorInfo, name);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FlatMapOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FlatMapOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FlatMapOperatorBase.java
index 0de236e..89575b6 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FlatMapOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/FlatMapOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericFlatMap;
+import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -28,9 +28,9 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericFlatMap
+ * @see org.apache.flink.api.common.functions.FlatMapFunction
*/
-public class FlatMapOperatorBase<IN, OUT, FT extends GenericFlatMap<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
+public class FlatMapOperatorBase<IN, OUT, FT extends FlatMapFunction<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
public FlatMapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
super(udf, operatorInfo, name);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
index a24826a..ac55489 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
@@ -19,8 +19,9 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -30,9 +31,9 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericGroupReduce
+ * @see org.apache.flink.api.common.functions.GroupReduceFunction
*/
-public class GroupReduceOperatorBase<IN, OUT, FT extends GenericGroupReduce<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
+public class GroupReduceOperatorBase<IN, OUT, FT extends GroupReduceFunction<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
/**
* The ordering for the order inside a reduce group.
@@ -91,15 +92,15 @@ public class GroupReduceOperatorBase<IN, OUT, FT extends GenericGroupReduce<IN,
/**
* Marks the group reduce operation as combinable. Combinable operations may pre-reduce the
* data before the actual group reduce operations. Combinable user-defined functions
- * must implement the interface {@link GenericCombine}.
+ * must implement the interface {@link org.apache.flink.api.common.functions.FlatCombineFunction}.
*
* @param combinable Flag to mark the group reduce operation as combinable.
*/
public void setCombinable(boolean combinable) {
// sanity check
- if (combinable && !GenericCombine.class.isAssignableFrom(this.userFunction.getUserCodeClass())) {
- throw new IllegalArgumentException("Cannot set a UDF as combinable if it does not implement the interface " +
- GenericCombine.class.getName());
+ if (combinable && !FlatCombineFunction.class.isAssignableFrom(this.userFunction.getUserCodeClass())) {
+ throw new IllegalArgumentException("Cannot set a UDF as combinable if it does not implement the interface " +
+ FlatCombineFunction.class.getName());
} else {
this.combinable = combinable;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
index b4eeeaa..2ce0529 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputOperator;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -27,9 +27,9 @@ import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
/**
- * @see GenericJoiner
+ * @see org.apache.flink.api.common.functions.FlatJoinFunction
*/
-public class JoinOperatorBase<IN1, IN2, OUT, FT extends GenericJoiner<IN1, IN2, OUT>> extends DualInputOperator<IN1, IN2, OUT, FT>
+public class JoinOperatorBase<IN1, IN2, OUT, FT extends FlatJoinFunction<IN1, IN2, OUT>> extends DualInputOperator<IN1, IN2, OUT, FT>
{
public JoinOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
super(udf, operatorInfo, keyPositions1, keyPositions2, name);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java
index efd8fa9..26fde05 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -33,7 +33,7 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
* @param <OUT> The result type.
* @param <FT> The type of the user-defined function.
*/
-public class MapOperatorBase<IN, OUT, FT extends GenericMap<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
+public class MapOperatorBase<IN, OUT, FT extends MapFunction<IN, OUT>> extends SingleInputOperator<IN, OUT, FT> {
public MapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
super(udf, operatorInfo, name);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
index 62996ea..e6c435f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
@@ -19,7 +19,7 @@
package org.apache.flink.api.common.operators.base;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
@@ -31,12 +31,12 @@ import org.apache.flink.api.common.operators.util.UserCodeWrapper;
* Base data flow operator for Reduce user-defined functions. Accepts reduce functions
* and key positions. The key positions are expected in the flattened common data model.
*
- * @see GenericReduce
+ * @see org.apache.flink.api.common.functions.ReduceFunction
*
* @param <T> The type (parameters and return type) of the reduce function.
* @param <FT> The type of the reduce function.
*/
-public class ReduceOperatorBase<T, FT extends GenericReduce<T>> extends SingleInputOperator<T, T, FT> {
+public class ReduceOperatorBase<T, FT extends ReduceFunction<T>> extends SingleInputOperator<T, T, FT> {
/**
* Creates a grouped reduce data flow operator.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
index 2ebae24..7d6495b 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/util/OperatorUtil.java
@@ -25,11 +25,11 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.io.FileInputFormat;
import org.apache.flink.api.common.io.FileOutputFormat;
import org.apache.flink.api.common.io.InputFormat;
@@ -56,10 +56,10 @@ public class OperatorUtil {
static {
STUB_CONTRACTS.put(GenericCollectorMap.class, CollectorMapOperatorBase.class);
- STUB_CONTRACTS.put(GenericGroupReduce.class, GroupReduceOperatorBase.class);
- STUB_CONTRACTS.put(GenericCoGrouper.class, CoGroupOperatorBase.class);
- STUB_CONTRACTS.put(GenericCrosser.class, CrossOperatorBase.class);
- STUB_CONTRACTS.put(GenericJoiner.class, JoinOperatorBase.class);
+ STUB_CONTRACTS.put(GroupReduceFunction.class, GroupReduceOperatorBase.class);
+ STUB_CONTRACTS.put(CoGroupFunction.class, CoGroupOperatorBase.class);
+ STUB_CONTRACTS.put(CrossFunction.class, CrossOperatorBase.class);
+ STUB_CONTRACTS.put(FlatJoinFunction.class, JoinOperatorBase.class);
STUB_CONTRACTS.put(FileInputFormat.class, GenericDataSourceBase.class);
STUB_CONTRACTS.put(FileOutputFormat.class, GenericDataSinkBase.class);
STUB_CONTRACTS.put(InputFormat.class, GenericDataSourceBase.class);
@@ -67,7 +67,7 @@ public class OperatorUtil {
}
/**
- * Returns the associated {@link Operator} type for the given {@link org.apache.flink.api.common.functions.Function} class.
+ * Returns the associated {@link Operator} type for the given {@link org.apache.flink.api.common.functions.RichFunction} class.
*
* @param stubClass
* the stub class
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
index 647ceab..30091ab 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/util/OperatorUtilTest.java
@@ -21,12 +21,12 @@ package org.apache.flink.api.common.operators.util;
import static org.junit.Assert.assertEquals;
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.io.DelimitedInputFormat;
import org.apache.flink.api.common.io.FileOutputFormat;
import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
@@ -36,7 +36,6 @@ import org.apache.flink.api.common.operators.base.GenericDataSinkBase;
import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.base.JoinOperatorBase;
-import org.apache.flink.api.common.operators.util.OperatorUtil;
import org.apache.flink.types.IntValue;
import org.junit.Test;
@@ -85,7 +84,7 @@ public class OperatorUtilTest {
*/
@Test
public void getContractClassShouldReturnNullForStub() {
- final Class<?> result = OperatorUtil.getContractClass(Function.class);
+ final Class<?> result = OperatorUtil.getContractClass(RichFunction.class);
assertEquals(null, result);
}
@@ -116,13 +115,13 @@ public class OperatorUtilTest {
assertEquals(GenericDataSourceBase.class, result);
}
- static abstract class CoGrouper implements GenericCoGrouper<IntValue, IntValue, IntValue> {}
+ static abstract class CoGrouper implements CoGroupFunction<IntValue, IntValue, IntValue> {}
- static abstract class Crosser implements GenericCrosser<IntValue, IntValue, IntValue> {}
+ static abstract class Crosser implements CrossFunction<IntValue, IntValue, IntValue> {}
static abstract class Mapper implements GenericCollectorMap<IntValue, IntValue> {}
- static abstract class Matcher implements GenericJoiner<IntValue, IntValue, IntValue> {}
+ static abstract class Matcher implements FlatJoinFunction<IntValue, IntValue, IntValue> {}
- static abstract class Reducer implements GenericGroupReduce<IntValue, IntValue> {}
+ static abstract class Reducer implements GroupReduceFunction<IntValue, IntValue> {}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/pom.xml b/flink-examples/flink-java-examples/pom.xml
index 549e95b..ea0db5d 100644
--- a/flink-examples/flink-java-examples/pom.xml
+++ b/flink-examples/flink-java-examples/pom.xml
@@ -317,7 +317,6 @@ under the License.
</executions>
</plugin>
-
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/clustering/KMeans.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/clustering/KMeans.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/clustering/KMeans.java
index 4c22db1..8767aca 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/clustering/KMeans.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/clustering/KMeans.java
@@ -21,8 +21,9 @@ package org.apache.flink.example.java.clustering;
import java.io.Serializable;
import java.util.Collection;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
@@ -197,7 +198,7 @@ public class KMeans {
// *************************************************************************
/** Converts a Tuple2<Double,Double> into a Point. */
- public static final class TuplePointConverter extends MapFunction<Tuple2<Double, Double>, Point> {
+ public static final class TuplePointConverter implements MapFunction<Tuple2<Double, Double>, Point> {
@Override
public Point map(Tuple2<Double, Double> t) throws Exception {
@@ -206,7 +207,7 @@ public class KMeans {
}
/** Converts a Tuple3<Integer, Double,Double> into a Centroid. */
- public static final class TupleCentroidConverter extends MapFunction<Tuple3<Integer, Double, Double>, Centroid> {
+ public static final class TupleCentroidConverter implements MapFunction<Tuple3<Integer, Double, Double>, Centroid> {
@Override
public Centroid map(Tuple3<Integer, Double, Double> t) throws Exception {
@@ -215,7 +216,7 @@ public class KMeans {
}
/** Determines the closest cluster center for a data point. */
- public static final class SelectNearestCenter extends MapFunction<Point, Tuple2<Integer, Point>> {
+ public static final class SelectNearestCenter extends RichMapFunction<Point, Tuple2<Integer, Point>> {
private Collection<Centroid> centroids;
/** Reads the centroid values from a broadcast variable into a collection. */
@@ -248,7 +249,7 @@ public class KMeans {
}
/** Appends a count variable to the tuple. */
- public static final class CountAppender extends MapFunction<Tuple2<Integer, Point>, Tuple3<Integer, Point, Long>> {
+ public static final class CountAppender implements MapFunction<Tuple2<Integer, Point>, Tuple3<Integer, Point, Long>> {
@Override
public Tuple3<Integer, Point, Long> map(Tuple2<Integer, Point> t) {
@@ -257,7 +258,7 @@ public class KMeans {
}
/** Sums and counts point coordinates. */
- public static final class CentroidAccumulator extends ReduceFunction<Tuple3<Integer, Point, Long>> {
+ public static final class CentroidAccumulator implements ReduceFunction<Tuple3<Integer, Point, Long>> {
@Override
public Tuple3<Integer, Point, Long> reduce(Tuple3<Integer, Point, Long> val1, Tuple3<Integer, Point, Long> val2) {
@@ -266,7 +267,7 @@ public class KMeans {
}
/** Computes new centroid from coordinate sum and count of points. */
- public static final class CentroidAverager extends MapFunction<Tuple3<Integer, Point, Long>, Centroid> {
+ public static final class CentroidAverager implements MapFunction<Tuple3<Integer, Point, Long>, Centroid> {
@Override
public Centroid map(Tuple3<Integer, Point, Long> value) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/ConnectedComponents.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/ConnectedComponents.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/ConnectedComponents.java
index b71347a..0d38e06 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/ConnectedComponents.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/ConnectedComponents.java
@@ -20,10 +20,11 @@
package org.apache.flink.example.java.graph;
import org.apache.flink.api.common.ProgramDescription;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsFirst;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsSecond;
@@ -127,7 +128,7 @@ public class ConnectedComponents implements ProgramDescription {
* Function that turns a value into a 2-tuple where both fields are that value.
*/
@ConstantFields("0 -> 0,1")
- public static final class DuplicateValue<T> extends MapFunction<T, Tuple2<T, T>> {
+ public static final class DuplicateValue<T> implements MapFunction<T, Tuple2<T, T>> {
@Override
public Tuple2<T, T> map(T vertex) {
@@ -138,7 +139,7 @@ public class ConnectedComponents implements ProgramDescription {
/**
* Undirected edges by emitting for each input edge the input edges itself and an inverted version.
*/
- public static final class UndirectEdge extends FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class UndirectEdge implements FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
Tuple2<Long, Long> invertedEdge = new Tuple2<Long, Long>();
@Override
@@ -157,7 +158,7 @@ public class ConnectedComponents implements ProgramDescription {
*/
@ConstantFieldsFirst("1 -> 0")
@ConstantFieldsSecond("1 -> 1")
- public static final class NeighborWithComponentIDJoin extends JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class NeighborWithComponentIDJoin implements JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public Tuple2<Long, Long> join(Tuple2<Long, Long> vertexWithComponent, Tuple2<Long, Long> edge) {
@@ -165,11 +166,10 @@ public class ConnectedComponents implements ProgramDescription {
}
}
- /**
- * The input is nested tuples ( (vertex-id, candidate-component) , (vertex-id, current-component) )
- */
+
+
@ConstantFieldsFirst("0")
- public static final class ComponentIdFilter extends JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class ComponentIdFilter implements FlatJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public void join(Tuple2<Long, Long> candidate, Tuple2<Long, Long> old, Collector<Tuple2<Long, Long>> out) {
@@ -177,10 +177,10 @@ public class ConnectedComponents implements ProgramDescription {
out.collect(candidate);
}
}
- @Override
- public Tuple2<Long, Long> join(Tuple2<Long, Long> first, Tuple2<Long, Long> second) { return null; }
}
+
+
@Override
public String getDescription() {
return "Parameters: <vertices-path> <edges-path> <result-path> <max-number-of-iterations>";
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
index fba18fc..2d794bd 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
@@ -22,10 +22,10 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
import org.apache.flink.api.java.DataSet;
@@ -119,7 +119,7 @@ public class EnumTrianglesBasic {
// *************************************************************************
/** Converts a Tuple2 into an Edge */
- public static class TupleEdgeConverter extends MapFunction<Tuple2<Integer, Integer>, Edge> {
+ public static class TupleEdgeConverter implements MapFunction<Tuple2<Integer, Integer>, Edge> {
private final Edge outEdge = new Edge();
@Override
@@ -130,7 +130,7 @@ public class EnumTrianglesBasic {
}
/** Projects an edge (pair of vertices) such that the id of the first is smaller than the id of the second. */
- private static class EdgeByIdProjector extends MapFunction<Edge, Edge> {
+ private static class EdgeByIdProjector implements MapFunction<Edge, Edge> {
@Override
public Edge map(Edge inEdge) throws Exception {
@@ -149,7 +149,7 @@ public class EnumTrianglesBasic {
* The first vertex of a triad is the shared vertex, the second and third vertex are ordered by vertexId.
* Assumes that input edges share the first vertex and are in ascending order of the second vertex.
*/
- private static class TriadBuilder extends GroupReduceFunction<Edge, Triad> {
+ private static class TriadBuilder implements GroupReduceFunction<Edge, Triad> {
private final List<Integer> vertices = new ArrayList<Integer>();
private final Triad outTriad = new Triad();
@@ -180,7 +180,7 @@ public class EnumTrianglesBasic {
}
/** Filters triads (three vertices connected by two edges) without a closing third edge. */
- private static class TriadFilter extends JoinFunction<Triad, Edge, Triad> {
+ private static class TriadFilter implements JoinFunction<Triad, Edge, Triad> {
@Override
public Triad join(Triad triad, Edge edge) throws Exception {
[28/39] [FLINK-1023] Switch group-at-a-time function to
java.lang.Iterable (from java.util.Iterator) Iterables over transient data
throw an TraversableOnceException when iterated over again.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
index a325e33..a29d4e0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
@@ -16,10 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.drivers;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.functions.GroupReduceFunction;
@@ -152,11 +150,10 @@ public class GroupReduceDriverTest {
public static final class ConcatSumReducer extends RichGroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
@Override
- public void reduce(Iterator<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
- Tuple2<String, Integer> current = values.next();
+ public void reduce(Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
+ Tuple2<String, Integer> current = new Tuple2<String, Integer>("", 0);
- while (values.hasNext()) {
- Tuple2<String, Integer> next = values.next();
+ for (Tuple2<String, Integer> next : values) {
next.f0 = current.f0 + next.f0;
next.f1 = current.f1 + next.f1;
current = next;
@@ -169,11 +166,10 @@ public class GroupReduceDriverTest {
public static final class ConcatSumMutableReducer extends RichGroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
@Override
- public void reduce(Iterator<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
- Tuple2<StringValue, IntValue> current = values.next();
+ public void reduce(Iterable<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
+ Tuple2<StringValue, IntValue> current = new Tuple2<StringValue, IntValue>(new StringValue(""), new IntValue(0));
- while (values.hasNext()) {
- Tuple2<StringValue, IntValue> next = values.next();
+ for (Tuple2<StringValue, IntValue> next : values) {
next.f0.append(current.f0);
next.f1.setValue(current.f1.getValue() + next.f1.getValue());
current = next;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
index 7020f89..4b3c197 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/CombiningUnilateralSortMergerITCase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.sort;
import java.io.IOException;
@@ -26,13 +25,12 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
import org.apache.flink.configuration.Configuration;
@@ -273,7 +271,7 @@ public class CombiningUnilateralSortMergerITCase {
// --------------------------------------------------------------------------------------------
- public static class TestCountCombiner extends ReduceFunction {
+ public static class TestCountCombiner extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
private final IntValue count = new IntValue();
@@ -284,11 +282,11 @@ public class CombiningUnilateralSortMergerITCase {
@Override
- public void combine(Iterator<Record> values, Collector<Record> out) {
+ public void combine(Iterable<Record> values, Collector<Record> out) {
Record rec = null;
int cnt = 0;
- while (values.hasNext()) {
- rec = values.next();
+ for (Record next : values) {
+ rec = next;
cnt += rec.getField(1, IntValue.class).getValue();
}
@@ -298,7 +296,7 @@ public class CombiningUnilateralSortMergerITCase {
}
@Override
- public void reduce(Iterator<Record> values, Collector<Record> out) {}
+ public void reduce(Iterable<Record> values, Collector<Record> out) {}
@Override
public void open(Configuration parameters) throws Exception {
@@ -311,7 +309,7 @@ public class CombiningUnilateralSortMergerITCase {
}
}
- public static class TestCountCombiner2 extends ReduceFunction {
+ public static class TestCountCombiner2 extends RichGroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
public volatile boolean opened = false;
@@ -319,11 +317,11 @@ public class CombiningUnilateralSortMergerITCase {
public volatile boolean closed = false;
@Override
- public void combine(Iterator<Record> values, Collector<Record> out) {
+ public void combine(Iterable<Record> values, Collector<Record> out) {
Record rec = null;
int cnt = 0;
- while (values.hasNext()) {
- rec = values.next();
+ for (Record next : values) {
+ rec = next;
cnt += Integer.parseInt(rec.getField(1, TestData.Value.class).toString());
}
@@ -331,7 +329,7 @@ public class CombiningUnilateralSortMergerITCase {
}
@Override
- public void reduce(Iterator<Record> values, Collector<Record> out) {
+ public void reduce(Iterable<Record> values, Collector<Record> out) {
// yo, nothing, mon
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIteratorITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIteratorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIteratorITCase.java
index a66fb5d..48a4b91 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIteratorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeCoGroupIteratorITCase.java
@@ -116,8 +116,8 @@ public class SortMergeCoGroupIteratorITCase
final TestData.Key key = new TestData.Key();
while (iterator.next())
{
- Iterator<Record> iter1 = iterator.getValues1();
- Iterator<Record> iter2 = iterator.getValues2();
+ Iterator<Record> iter1 = iterator.getValues1().iterator();
+ Iterator<Record> iter2 = iterator.getValues2().iterator();
TestData.Value v1 = null;
TestData.Value v2 = null;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-runtime/src/test/java/org/apache/flink/runtime/util/KeyGroupedIteratorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/util/KeyGroupedIteratorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/util/KeyGroupedIteratorTest.java
index e9010d2..27fa540 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/util/KeyGroupedIteratorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/util/KeyGroupedIteratorTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.types.IntValue;
import org.apache.flink.types.Record;
import org.apache.flink.types.StringValue;
import org.apache.flink.util.MutableObjectIterator;
+import org.apache.flink.util.TraversableOnceException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -38,17 +39,15 @@ import org.junit.Test;
/**
* Test for the key grouped iterator, which advances in windows containing the same key and provides a sub-iterator
* over the records with the same key.
- *
*/
-public class KeyGroupedIteratorTest
-{
+public class KeyGroupedIteratorTest {
+
private MutableObjectIterator<Record> sourceIter; // the iterator that provides the input
private KeyGroupedIterator<Record> psi; // the grouping iterator, progressing in key steps
@Before
- public void setup()
- {
+ public void setup() {
final ArrayList<IntStringPair> source = new ArrayList<IntStringPair>();
// add elements to the source
@@ -91,8 +90,7 @@ public class KeyGroupedIteratorTest
}
@Test
- public void testNextKeyOnly() throws Exception
- {
+ public void testNextKeyOnly() throws Exception {
try {
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
@@ -131,6 +129,8 @@ public class KeyGroupedIteratorTest
try {
// Key 1, Value A
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -139,6 +139,8 @@ public class KeyGroupedIteratorTest
// Key 2, Value B
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -147,6 +149,8 @@ public class KeyGroupedIteratorTest
// Key 3, Values C, D
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -173,6 +177,8 @@ public class KeyGroupedIteratorTest
// Key 4, Values E, F, G
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -193,6 +199,8 @@ public class KeyGroupedIteratorTest
// Key 5, Values H, I, J, K, L
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -231,6 +239,7 @@ public class KeyGroupedIteratorTest
Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
+ Assert.assertNull(this.psi.getValues());
} catch (Exception e) {
e.printStackTrace();
Assert.fail("The test encountered an unexpected exception.");
@@ -244,12 +253,18 @@ public class KeyGroupedIteratorTest
// Progression only via nextKey() and hasNext() - Key 1, Value A
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
// Progression only through nextKey() - Key 2, Value B
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
// Progression first though haNext() and next(), then through hasNext() - Key 3, Values C, D
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
@@ -260,6 +275,8 @@ public class KeyGroupedIteratorTest
// Progression first via next() only, then hasNext() only Key 4, Values E, F, G
Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
@@ -270,6 +287,8 @@ public class KeyGroupedIteratorTest
Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
+ Assert.assertTrue(hasIterator(this.psi.getValues()));
+ Assert.assertFalse(hasIterator(this.psi.getValues()));
Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
// end
@@ -355,4 +374,14 @@ public class KeyGroupedIteratorTest
return string;
}
}
+
+ public boolean hasIterator(Iterable<?> iterable) {
+ try {
+ iterable.iterator();
+ return true;
+ }
+ catch (TraversableOnceException e) {
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/ReduceFunction.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/ReduceFunction.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/ReduceFunction.scala
index 6ea8cbf..de7b7d1 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/ReduceFunction.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/ReduceFunction.scala
@@ -19,10 +19,10 @@
package org.apache.flink.api.scala.functions
-import java.util.{ Iterator => JIterator }
-
import scala.Iterator
+import java.util.{Iterator => JIterator}
+
import org.apache.flink.api.scala.analysis.{UDTSerializer, FieldSelector, UDT}
import org.apache.flink.api.scala.analysis.UDF1
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CoGroupOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CoGroupOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CoGroupOperator.scala
index 4c26307..6a71bb7 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CoGroupOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CoGroupOperator.scala
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.scala.operators
import language.experimental.macros
@@ -54,7 +53,7 @@ class CoGroupDataSetWithWhereAndEqual[LeftIn, RightIn](val leftKeySelection: Lis
def flatMap[Out](fun: (Iterator[LeftIn], Iterator[RightIn]) => Iterator[Out]): DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out] = macro CoGroupMacros.flatMap[LeftIn, RightIn, Out]
}
-class NoKeyCoGroupBuilder(s: JCoGroupFunction) extends CoGroupOperator.Builder(new UserCodeObjectWrapper(s))
+class NoKeyCoGroupBuilder(s: JCoGroupFunction) extends CoGroupOperator.Builder(new UserCodeObjectWrapper(new CoGroupOperator.WrappingCoGroupFunction(s)))
object CoGroupMacros {
@@ -106,7 +105,9 @@ object CoGroupMacros {
implicit val leftInputUDT: UDT[LeftIn] = c.Expr[UDT[LeftIn]](createUdtLeftIn).splice
implicit val rightInputUDT: UDT[RightIn] = c.Expr[UDT[RightIn]](createUdtRightIn).splice
implicit val outputUDT: UDT[Out] = c.Expr[UDT[Out]](createUdtOut).splice
+
new CoGroupFunctionBase[LeftIn, RightIn, Out] {
+
override def coGroup(leftRecords: JIterator[Record], rightRecords: JIterator[Record], out: Collector[Record]) = {
val firstLeftRecord = leftIterator.initialize(leftRecords)
@@ -177,7 +178,9 @@ object CoGroupMacros {
implicit val leftInputUDT: UDT[LeftIn] = c.Expr[UDT[LeftIn]](createUdtLeftIn).splice
implicit val rightInputUDT: UDT[RightIn] = c.Expr[UDT[RightIn]](createUdtRightIn).splice
implicit val outputUDT: UDT[Out] = c.Expr[UDT[Out]](createUdtOut).splice
+
new CoGroupFunctionBase[LeftIn, RightIn, Out] {
+
override def coGroup(leftRecords: JIterator[Record], rightRecords: JIterator[Record], out: Collector[Record]) = {
val firstLeftRecord = leftIterator.initialize(leftRecords)
val firstRightRecord = rightIterator.initialize(rightRecords)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ReduceOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ReduceOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ReduceOperator.scala
index cf3a96c..f25b5a3 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ReduceOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/ReduceOperator.scala
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.scala.operators
import language.experimental.macros
@@ -191,7 +190,9 @@ object ReduceMacros {
implicit val outputUDT: UDT[Out] = c.Expr[UDT[Out]](createUdtOut).splice
new ReduceFunctionBase[In, Out] {
- override def reduce(records: JIterator[Record], out: Collector[Record]) = {
+ override def reduce(recordsIterable: JIterator[Record], out: Collector[Record]) = {
+ val records: JIterator[Record] = recordsIterable
+
if (records.hasNext) {
val firstRecord = reduceIterator.initialize(records)
reduceRecord.copyFrom(firstRecord, reduceForwardFrom, reduceForwardTo)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
index e76f86d..5497004 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorITCase.java
@@ -270,6 +270,7 @@ public class AccumulatorITCase extends RecordAPITestBase {
private void reduceInternal(Iterator<Record> records, Collector<Record> out) {
Record element = null;
int sum = 0;
+
while (records.hasNext()) {
element = records.next();
IntValue i = element.getField(1, IntValue.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorIterativeITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorIterativeITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorIterativeITCase.java
index bf6c37d..197a745 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorIterativeITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorIterativeITCase.java
@@ -124,11 +124,13 @@ public class AccumulatorIterativeITCase extends RecordAPITestBase {
}
@Override
- public void reduce(Iterator<Record> it, Collector<Record> out) {
+ public void reduce(Iterator<Record> records, Collector<Record> out) {
// Compute the sum
int sum = 0;
- while (it.hasNext()) {
- Integer value = Integer.parseInt(it.next().getField(0, StringValue.class).getValue());
+
+ while (records.hasNext()) {
+ Record r = records.next();
+ Integer value = Integer.parseInt(r.getField(0, StringValue.class).getValue());
sum += value;
testCounter.add(value);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/broadcastvars/KMeansIterativeNepheleITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/broadcastvars/KMeansIterativeNepheleITCase.java b/flink-tests/src/test/java/org/apache/flink/test/broadcastvars/KMeansIterativeNepheleITCase.java
index 4b3a4bf..2217679 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/broadcastvars/KMeansIterativeNepheleITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/broadcastvars/KMeansIterativeNepheleITCase.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.record.io.CsvInputFormat;
+import org.apache.flink.api.java.record.operators.ReduceOperator.WrappingReduceFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
import org.apache.flink.configuration.Configuration;
@@ -252,7 +253,7 @@ public class KMeansIterativeNepheleITCase extends RecordAPITestBase {
tailConfig.setOutputSerializer(outputSerializer);
// the udf
- tailConfig.setStubWrapper(new UserCodeObjectWrapper<RecomputeClusterCenter>(new RecomputeClusterCenter()));
+ tailConfig.setStubWrapper(new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingReduceFunction(new RecomputeClusterCenter())));
return tail;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
index 35aea4b..b442b33 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
@@ -16,12 +16,10 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
@@ -72,15 +70,13 @@ public class BulkIterationWithAllReducerITCase extends JavaProgramTestBase {
}
@Override
- public void reduce(Iterator<Integer> records, Collector<Integer> out) {
+ public void reduce(Iterable<Integer> records, Collector<Integer> out) {
if (bcValue == null) {
return;
}
final int x = bcValue;
- while (records.hasNext()) {
- int y = records.next();
-
+ for (Integer y : records) {
if (y > x) {
out.collect(y);
return;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsITCase.java
index 37695b2..f2a43a8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsITCase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.io.BufferedReader;
@@ -104,7 +103,8 @@ public class CoGroupConnectedComponentsITCase extends RecordAPITestBase {
long minimumComponentID = Long.MAX_VALUE;
while (candidates.hasNext()) {
- long candidateComponentID = candidates.next().getField(1, LongValue.class).getValue();
+ Record candidate = candidates.next();
+ long candidateComponentID = candidate.getField(1, LongValue.class).getValue();
if (candidateComponentID < minimumComponentID) {
minimumComponentID = candidateComponentID;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
index 49bdb26..c6175b2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.java.functions.RichCoGroupFunction;
@@ -116,17 +114,17 @@ public class CoGroupConnectedComponentsSecondITCase extends JavaProgramTestBase
public static final class MinIdAndUpdate extends RichCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
- public void coGroup(Iterator<Tuple2<Long, Long>> candidates, Iterator<Tuple2<Long, Long>> current, Collector<Tuple2<Long, Long>> out) {
- if (!current.hasNext()) {
+ public void coGroup(Iterable<Tuple2<Long, Long>> candidates, Iterable<Tuple2<Long, Long>> current, Collector<Tuple2<Long, Long>> out) {
+ if (!current.iterator().hasNext()) {
throw new RuntimeException("Error: Id not encountered before.");
}
- Tuple2<Long, Long> old = current.next();
+ Tuple2<Long, Long> old = current.iterator().next();
long minimumComponentID = Long.MAX_VALUE;
- while (candidates.hasNext()) {
- long candidateComponentID = candidates.next().f1;
+ for (Tuple2<Long, Long> candidate : candidates) {
+ long candidateComponentID = candidate.f1;
if (candidateComponentID < minimumComponentID) {
minimumComponentID = candidateComponentID;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
index d2669aa..fdfb321 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.java.functions.RichFlatMapFunction;
@@ -170,8 +168,8 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Long> values, Collector<Long> out) throws Exception {
- out.collect(values.next());
+ public void reduce(Iterable<Long> values, Collector<Long> out) {
+ out.collect(values.iterator().next());
}
}
@@ -204,21 +202,19 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@Override
- public void reduce(Iterator<Tuple2<Long, Long>> values,
- Collector<Tuple2<Long, Long>> out) throws Exception {
+ public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) {
+ Long vertexId = 0L;
+ Long minimumCompId = Long.MAX_VALUE;
- final Tuple2<Long, Long> first = values.next();
- final Long vertexId = first.f0;
- Long minimumCompId = first.f1;
-
- while ( values.hasNext() ) {
- Long candidateCompId = values.next().f1;
- if ( candidateCompId < minimumCompId ) {
+ for (Tuple2<Long, Long> value: values) {
+ vertexId = value.f0;
+ Long candidateCompId = value.f1;
+ if (candidateCompId < minimumCompId) {
minimumCompId = candidateCompId;
}
}
- resultVertex.setField(vertexId, 0);
- resultVertex.setField(minimumCompId, 1);
+ resultVertex.f0 = vertexId;
+ resultVertex.f1 = minimumCompId;
out.collect(resultVertex);
}
@@ -231,8 +227,8 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
@Override
public void flatMap(
Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> vertexWithNewAndOldId,
- Collector<Tuple2<Long, Long>> out) throws Exception {
-
+ Collector<Tuple2<Long, Long>> out)
+ {
if ( vertexWithNewAndOldId.f0.f1 < vertexWithNewAndOldId.f1.f1 ) {
out.collect(vertexWithNewAndOldId.f0);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTerminationTail.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTerminationTail.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTerminationTail.java
index cb25019..3dc0bdf 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTerminationTail.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTerminationTail.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTwoTails.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTwoTails.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTwoTails.java
index 6a487c4..2c777f5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTwoTails.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationTerminationWithTwoTails.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.io.Serializable;
@@ -110,6 +109,7 @@ public class IterationTerminationWithTwoTails extends RecordAPITestBase {
public void reduce(Iterator<Record> it, Collector<Record> out) {
// Compute the sum
int sum = 0;
+
while (it.hasNext()) {
sum += Integer.parseInt(it.next().getField(0, StringValue.class).getValue()) + 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithAllReducerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithAllReducerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithAllReducerITCase.java
index c81b32a..b3145dc 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithAllReducerITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithAllReducerITCase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithChainingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithChainingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithChainingITCase.java
index b62d85a..d00602f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithChainingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithChainingITCase.java
@@ -44,85 +44,81 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class IterationWithChainingITCase extends RecordAPITestBase {
- private static final String DATA_POINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n";
+ private static final String DATA_POINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n";
- private String dataPath;
- private String resultPath;
+ private String dataPath;
+ private String resultPath;
- public IterationWithChainingITCase(Configuration config) {
- super(config);
+ public IterationWithChainingITCase(Configuration config) {
+ super(config);
setTaskManagerNumSlots(DOP);
- }
+ }
- @Override
- protected void preSubmit() throws Exception {
- dataPath = createTempFile("data_points.txt", DATA_POINTS);
- resultPath = getTempFilePath("result");
- }
+ @Override
+ protected void preSubmit() throws Exception {
+ dataPath = createTempFile("data_points.txt", DATA_POINTS);
+ resultPath = getTempFilePath("result");
+ }
- @Override
- protected void postSubmit() throws Exception {
- compareResultsByLinesInMemory(DATA_POINTS, resultPath);
- }
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(DATA_POINTS, resultPath);
+ }
+ @Override
+ protected Plan getTestJob() {
+ Plan plan = getTestPlan(config.getInteger("ChainedMapperITCase#NoSubtasks", 1), dataPath, resultPath);
+ return plan;
+ }
- @Override
- protected Plan getTestJob() {
- Plan plan = getTestPlan(config.getInteger("ChainedMapperITCase#NoSubtasks", 1), dataPath, resultPath);
- return plan;
- }
+ @Parameters
+ public static Collection<Object[]> getConfigurations() {
+ Configuration config1 = new Configuration();
+ config1.setInteger("ChainedMapperITCase#NoSubtasks", DOP);
+ return toParameterList(config1);
+ }
- @Parameters
- public static Collection<Object[]> getConfigurations() {
- Configuration config1 = new Configuration();
- config1.setInteger("ChainedMapperITCase#NoSubtasks", DOP);
- return toParameterList(config1);
- }
+ public static final class IdentityMapper extends MapFunction implements Serializable {
- public static final class IdentityMapper extends MapFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
- private static final long serialVersionUID = 1L;
+ @Override
+ public void map(Record rec, Collector<Record> out) {
+ out.collect(rec);
+ }
+ }
- @Override
- public void map(Record rec, Collector<Record> out) {
- out.collect(rec);
- }
- }
+ public static final class DummyReducer extends ReduceFunction implements Serializable {
- public static final class DummyReducer extends ReduceFunction implements Serializable {
+ private static final long serialVersionUID = 1L;
- private static final long serialVersionUID = 1L;
+ @Override
+ public void reduce(Iterator<Record> it, Collector<Record> out) {
+ while (it.hasNext()) {
+ out.collect(it.next());
+ }
+ }
+ }
- @Override
- public void reduce(Iterator<Record> it, Collector<Record> out) {
- while (it.hasNext()) {
- out.collect(it.next());
- }
- }
- }
+ static Plan getTestPlan(int numSubTasks, String input, String output) {
- static Plan getTestPlan(int numSubTasks, String input, String output) {
+ FileDataSource initialInput = new FileDataSource(new PointInFormat(), input, "Input");
+ initialInput.setDegreeOfParallelism(1);
- FileDataSource initialInput = new FileDataSource(new PointInFormat(), input, "Input");
- initialInput.setDegreeOfParallelism(1);
+ BulkIteration iteration = new BulkIteration("Loop");
+ iteration.setInput(initialInput);
+ iteration.setMaximumNumberOfIterations(2);
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(initialInput);
- iteration.setMaximumNumberOfIterations(2);
+ ReduceOperator dummyReduce = ReduceOperator.builder(new DummyReducer(), IntValue.class, 0).input(iteration.getPartialSolution())
+ .name("Reduce something").build();
- ReduceOperator dummyReduce = ReduceOperator.builder(new DummyReducer(), IntValue.class, 0)
- .input(iteration.getPartialSolution())
- .name("Reduce something")
- .build();
+ MapOperator dummyMap = MapOperator.builder(new IdentityMapper()).input(dummyReduce).build();
+ iteration.setNextPartialSolution(dummyMap);
+ FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");
- MapOperator dummyMap = MapOperator.builder(new IdentityMapper()).input(dummyReduce).build();
- iteration.setNextPartialSolution(dummyMap);
-
- FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");
-
- Plan plan = new Plan(finalResult, "Iteration with chained map test");
- plan.setDefaultParallelism(numSubTasks);
- return plan;
- }
+ Plan plan = new Plan(finalResult, "Iteration with chained map test");
+ plan.setDefaultParallelism(numSubTasks);
+ return plan;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithUnionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithUnionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithUnionITCase.java
index aa83e9b..2a130ed 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithUnionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/IterationWithUnionITCase.java
@@ -16,43 +16,28 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative;
import java.io.Serializable;
-import java.util.Collection;
-import java.util.Iterator;
-import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.java.record.functions.MapFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
-import org.apache.flink.api.java.record.operators.BulkIteration;
-import org.apache.flink.api.java.record.operators.FileDataSink;
-import org.apache.flink.api.java.record.operators.FileDataSource;
-import org.apache.flink.api.java.record.operators.MapOperator;
-import org.apache.flink.configuration.Configuration;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.IterativeDataSet;
import org.apache.flink.test.recordJobs.kmeans.udfs.PointInFormat;
import org.apache.flink.test.recordJobs.kmeans.udfs.PointOutFormat;
-import org.apache.flink.test.util.RecordAPITestBase;
+import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-@RunWith(Parameterized.class)
-public class IterationWithUnionITCase extends RecordAPITestBase {
+public class IterationWithUnionITCase extends JavaProgramTestBase {
private static final String DATAPOINTS = "0|50.90|16.20|72.08|\n" + "1|73.65|61.76|62.89|\n" + "2|61.73|49.95|92.74|\n";
protected String dataPath;
protected String resultPath;
-
- public IterationWithUnionITCase(Configuration config) {
- super(config);
- setTaskManagerNumSlots(DOP);
- }
@Override
protected void preSubmit() throws Exception {
@@ -66,54 +51,36 @@ public class IterationWithUnionITCase extends RecordAPITestBase {
}
@Override
- protected Plan getTestJob() {
- return getPlan(config.getInteger("IterationWithUnionITCase#NumSubtasks", 1), dataPath, resultPath);
- }
-
- @Parameters
- public static Collection<Object[]> getConfigurations() {
- Configuration config1 = new Configuration();
- config1.setInteger("IterationWithUnionITCase#NumSubtasks", DOP);
-
- return toParameterList(config1);
- }
-
- private static Plan getPlan(int numSubTasks, String input, String output) {
- FileDataSource initialInput = new FileDataSource(new PointInFormat(), input, "Input");
- initialInput.setDegreeOfParallelism(1);
+ protected void testProgram() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- BulkIteration iteration = new BulkIteration("Loop");
- iteration.setInput(initialInput);
- iteration.setMaximumNumberOfIterations(2);
-
- @SuppressWarnings("unchecked")
- MapOperator map2 = MapOperator.builder(new IdentityMapper()).input(iteration.getPartialSolution(), iteration.getPartialSolution()).name("map").build();
+ DataSet<Record> initialInput = env.readFile(new PointInFormat(), this.dataPath).setParallelism(1);
- iteration.setNextPartialSolution(map2);
-
- FileDataSink finalResult = new FileDataSink(new PointOutFormat(), output, iteration, "Output");
-
- Plan plan = new Plan(finalResult, "Iteration with union test");
- plan.setDefaultParallelism(numSubTasks);
- return plan;
+ IterativeDataSet<Record> iteration = initialInput.iterate(2);
+
+ DataSet<Record> result = iteration.union(iteration).map(new IdentityMapper());
+
+ iteration.closeWith(result).write(new PointOutFormat(), this.resultPath);
+
+ env.execute();
}
- static final class IdentityMapper extends MapFunction implements Serializable {
+ static final class IdentityMapper implements MapFunction<Record, Record>, Serializable {
private static final long serialVersionUID = 1L;
@Override
- public void map(Record rec, Collector<Record> out) {
- out.collect(rec);
+ public Record map(Record rec) {
+ return rec;
}
}
- static final class DummyReducer extends ReduceFunction implements Serializable {
+ static final class DummyReducer implements GroupReduceFunction<Record, Record>, Serializable {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Record> it, Collector<Record> out) {
- while (it.hasNext()) {
- out.collect(it.next());
+ public void reduce(Iterable<Record> it, Collector<Record> out) {
+ for (Record r : it) {
+ out.collect(r);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
index 4c8177a..9382708 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
@@ -16,34 +16,27 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative.aggregators;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.types.LongValue;
import org.apache.flink.util.Collector;
-
import org.junit.Assert;
-
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
-
/**
- *
- * Connected Components test case that uses a parametrizable aggregator
- *
+ * Connected Components test case that uses a parameterizable aggregator
*/
public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaProgramTestBase {
@@ -147,7 +140,8 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
}
}
- public static final class NeighborWithComponentIDJoin extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class NeighborWithComponentIDJoin implements JoinFunction
+ <Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
@@ -163,24 +157,23 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
public static final class MinimumReduce extends RichGroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
- final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
+
+ private final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@Override
- public void reduce(Iterator<Tuple2<Long, Long>> values,
- Collector<Tuple2<Long, Long>> out) throws Exception {
-
- final Tuple2<Long, Long> first = values.next();
- final Long vertexId = first.f0;
- Long minimumCompId = first.f1;
+ public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) {
+ Long vertexId = 0L;
+ Long minimumCompId = Long.MAX_VALUE;
- while (values.hasNext()) {
- Long candidateCompId = values.next().f1;
+ for (Tuple2<Long, Long> value: values) {
+ vertexId = value.f0;
+ Long candidateCompId = value.f1;
if (candidateCompId < minimumCompId) {
minimumCompId = candidateCompId;
}
}
- resultVertex.setField(vertexId, 0);
- resultVertex.setField(minimumCompId, 1);
+ resultVertex.f0 = vertexId;
+ resultVertex.f1 = minimumCompId;
out.collect(resultVertex);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
index 104c3df..039d64e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
@@ -16,11 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative.aggregators;
import java.util.ArrayList;
-import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
@@ -158,21 +156,19 @@ public class ConnectedComponentsWithParametrizableConvergenceITCase extends Java
final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@Override
- public void reduce(Iterator<Tuple2<Long, Long>> values,
- Collector<Tuple2<Long, Long>> out) throws Exception {
-
- final Tuple2<Long, Long> first = values.next();
- final Long vertexId = first.f0;
- Long minimumCompId = first.f1;
+ public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) {
+ Long vertexId = 0L;
+ Long minimumCompId = Long.MAX_VALUE;
- while (values.hasNext()) {
- Long candidateCompId = values.next().f1;
+ for (Tuple2<Long, Long> value: values) {
+ vertexId = value.f0;
+ Long candidateCompId = value.f1;
if (candidateCompId < minimumCompId) {
minimumCompId = candidateCompId;
}
}
- resultVertex.setField(vertexId, 0);
- resultVertex.setField(minimumCompId, 1);
+ resultVertex.f0 = vertexId;
+ resultVertex.f1 = minimumCompId;
out.collect(resultVertex);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/ConnectedComponentsNepheleITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/ConnectedComponentsNepheleITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/ConnectedComponentsNepheleITCase.java
index 3060b68..4fd22a3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/ConnectedComponentsNepheleITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/ConnectedComponentsNepheleITCase.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
+import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -31,6 +32,8 @@ import org.apache.flink.api.java.record.functions.MapFunction;
import org.apache.flink.api.java.record.io.CsvInputFormat;
import org.apache.flink.api.java.record.io.CsvOutputFormat;
import org.apache.flink.api.java.record.io.FileOutputFormat;
+import org.apache.flink.api.java.record.operators.ReduceOperator.WrappingReduceFunction;
+import org.apache.flink.api.java.record.operators.ReduceOperator.WrappingClassReduceFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
@@ -317,7 +320,7 @@ public class ConnectedComponentsNepheleITCase extends RecordAPITestBase {
intermediateConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
intermediateConfig.setDriverComparator(comparator, 0);
intermediateConfig.setStubWrapper(
- new UserCodeClassWrapper<MinimumComponentIDReduce>(MinimumComponentIDReduce.class));
+ new UserCodeObjectWrapper<WrappingReduceFunction>(new WrappingClassReduceFunction(MinimumComponentIDReduce.class)));
}
return intermediate;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/IterationWithChainingNepheleITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/IterationWithChainingNepheleITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/IterationWithChainingNepheleITCase.java
index 448b7bd..5a6e4f5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/IterationWithChainingNepheleITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/IterationWithChainingNepheleITCase.java
@@ -19,13 +19,12 @@
package org.apache.flink.test.iterative.nephele;
import java.util.Collection;
-import java.util.Iterator;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.record.functions.MapFunction;
-import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.io.FileOutputFormat;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
@@ -273,14 +272,14 @@ public class IterationWithChainingNepheleITCase extends RecordAPITestBase {
}
}
- public static final class DummyReducer extends ReduceFunction {
+ public static final class DummyReducer implements GroupReduceFunction<Record, Record> {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Record> it, Collector<Record> out) {
- while (it.hasNext()) {
- out.collect(it.next());
+ public void reduce(Iterable<Record> it, Collector<Record> out) {
+ for (Record r :it) {
+ out.collect(r);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
index 1ec0eb4..cd6a89d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative.nephele.customdanglingpagerank;
import java.util.Iterator;
@@ -83,11 +82,13 @@ public class CustomCompensatableDotProductCoGroup extends AbstractRichFunction i
}
@Override
- public void coGroup(Iterator<VertexWithRankAndDangling> currentPageRankIterator, Iterator<VertexWithRank> partialRanks,
+ public void coGroup(Iterable<VertexWithRankAndDangling> currentPageRankIterable, Iterable<VertexWithRank> partialRanks,
Collector<VertexWithRankAndDangling> collector)
{
+ final Iterator<VertexWithRankAndDangling> currentPageRankIterator = currentPageRankIterable.iterator();
+
if (!currentPageRankIterator.hasNext()) {
- long missingVertex = partialRanks.next().getVertexID();
+ long missingVertex = partialRanks.iterator().next().getVertexID();
throw new IllegalStateException("No current page rank for vertex [" + missingVertex + "]!");
}
@@ -95,8 +96,8 @@ public class CustomCompensatableDotProductCoGroup extends AbstractRichFunction i
long edges = 0;
double summedRank = 0;
- while (partialRanks.hasNext()) {
- summedRank += partialRanks.next().getRank();
+ for (VertexWithRank pr :partialRanks) {
+ summedRank += pr.getRank();
edges++;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
index d83b33b..8d92c59 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
@@ -29,6 +29,7 @@ import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.Vert
import org.apache.flink.test.iterative.nephele.danglingpagerank.PageRankStats;
import org.apache.flink.util.Collector;
+@SuppressWarnings("deprecation")
public class CustomCompensatingMap extends AbstractRichFunction implements GenericCollectorMap<VertexWithRankAndDangling, VertexWithRankAndDangling> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
index 1e08a9f..8ec8403 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
@@ -35,12 +35,14 @@ public class CustomRankCombiner extends AbstractRichFunction implements GroupRed
private final VertexWithRank accumulator = new VertexWithRank();
@Override
- public void reduce(Iterator<VertexWithRank> records, Collector<VertexWithRank> out) throws Exception {
+ public void reduce(Iterable<VertexWithRank> records, Collector<VertexWithRank> out) throws Exception {
throw new UnsupportedOperationException();
}
@Override
- public void combine(Iterator<VertexWithRank> records, Collector<VertexWithRank> out) throws Exception {
+ public void combine(Iterable<VertexWithRank> recordsIterable, Collector<VertexWithRank> out) throws Exception {
+ final Iterator<VertexWithRank> records = recordsIterable.iterator();
+
VertexWithRank next = records.next();
this.accumulator.setVertexID(next.getVertexID());
double rank = next.getRank();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/danglingpagerank/CompensatableDotProductCoGroup.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/danglingpagerank/CompensatableDotProductCoGroup.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/danglingpagerank/CompensatableDotProductCoGroup.java
index ab3dea9..c51cf96 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/danglingpagerank/CompensatableDotProductCoGroup.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/danglingpagerank/CompensatableDotProductCoGroup.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.iterative.nephele.danglingpagerank;
import java.util.Iterator;
@@ -102,7 +101,8 @@ public class CompensatableDotProductCoGroup extends CoGroupFunction {
long edges = 0;
double summedRank = 0;
while (partialRanks.hasNext()) {
- summedRank += partialRanks.next().getField(1, doubleInstance).getValue();
+ Record pr = partialRanks.next();
+ summedRank += pr.getField(1, doubleInstance).getValue();
edges++;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index 3749c1d..d59d721 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -21,7 +21,6 @@ package org.apache.flink.test.javaApiOperators;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collection;
-import java.util.Iterator;
import java.util.LinkedList;
import org.apache.flink.api.common.functions.CoGroupFunction;
@@ -308,21 +307,19 @@ public class CoGroupITCase extends JavaProgramTestBase {
@Override
public void coGroup(
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> first,
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> second,
- Collector<Tuple2<Integer, Integer>> out) throws Exception {
-
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> first,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> second,
+ Collector<Tuple2<Integer, Integer>> out)
+ {
int sum = 0;
int id = 0;
- while(first.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = first.next();
+ for ( Tuple5<Integer, Long, Integer, String, Long> element : first ) {
sum += element.f2;
id = element.f0;
}
- while(second.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = second.next();
+ for ( Tuple5<Integer, Long, Integer, String, Long> element : second ) {
sum += element.f2;
id = element.f0;
}
@@ -336,27 +333,22 @@ public class CoGroupITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void coGroup(Iterator<CustomType> first,
- Iterator<CustomType> second, Collector<CustomType> out)
- throws Exception {
+ public void coGroup(Iterable<CustomType> first, Iterable<CustomType> second, Collector<CustomType> out) {
CustomType o = new CustomType(0,0,"test");
- while(first.hasNext()) {
- CustomType element = first.next();
+ for ( CustomType element : first ) {
o.myInt = element.myInt;
o.myLong += element.myLong;
}
- while(second.hasNext()) {
- CustomType element = second.next();
+ for ( CustomType element : second ) {
o.myInt = element.myInt;
o.myLong += element.myLong;
}
out.collect(o);
}
-
}
public static class MixedCoGroup implements CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, CustomType, Tuple3<Integer, Long, String>> {
@@ -365,21 +357,19 @@ public class CoGroupITCase extends JavaProgramTestBase {
@Override
public void coGroup(
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> first,
- Iterator<CustomType> second,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> first,
+ Iterable<CustomType> second,
Collector<Tuple3<Integer, Long, String>> out) throws Exception {
long sum = 0;
int id = 0;
- while(first.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = first.next();
+ for ( Tuple5<Integer, Long, Integer, String, Long> element : first ) {
sum += element.f0;
id = element.f2;
}
- while(second.hasNext()) {
- CustomType element = second.next();
+ for (CustomType element : second) {
id = element.myInt;
sum += element.myLong;
}
@@ -394,20 +384,18 @@ public class CoGroupITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void coGroup(Iterator<CustomType> first,
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> second,
- Collector<CustomType> out) throws Exception {
-
+ public void coGroup(Iterable<CustomType> first,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> second,
+ Collector<CustomType> out)
+ {
CustomType o = new CustomType(0,0,"test");
- while(first.hasNext()) {
- CustomType element = first.next();
+ for (CustomType element : first) {
o.myInt = element.myInt;
o.myLong += element.myLong;
}
- while(second.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = second.next();
+ for (Tuple5<Integer, Long, Integer, String, Long> element : second) {
o.myInt = element.f2;
o.myLong += element.f0;
}
@@ -423,14 +411,14 @@ public class CoGroupITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void coGroup(Iterator<Tuple3<Integer, Long, String>> first,
- Iterator<Tuple3<Integer, Long, String>> second,
- Collector<Tuple3<Integer, Long, String>> out) throws Exception {
-
- while(first.hasNext()) {
- Tuple3<Integer, Long, String> element = first.next();
- if(element.f0 < 6)
+ public void coGroup(Iterable<Tuple3<Integer, Long, String>> first,
+ Iterable<Tuple3<Integer, Long, String>> second,
+ Collector<Tuple3<Integer, Long, String>> out)
+ {
+ for (Tuple3<Integer, Long, String> element : first) {
+ if(element.f0 < 6) {
out.collect(element);
+ }
}
}
}
@@ -441,20 +429,16 @@ public class CoGroupITCase extends JavaProgramTestBase {
@Override
public void coGroup(
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> first,
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> second,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> first,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> second,
Collector<Tuple5<Integer, Long, Integer, String, Long>> out)
- throws Exception {
-
- while(second.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = second.next();
- if(element.f0 < 4)
+ {
+ for (Tuple5<Integer, Long, Integer, String, Long> element : second) {
+ if(element.f0 < 4) {
out.collect(element);
+ }
}
-
}
-
-
}
public static class Tuple5CoGroupBC extends RichCoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Integer, Integer>> {
@@ -477,21 +461,19 @@ public class CoGroupITCase extends JavaProgramTestBase {
@Override
public void coGroup(
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> first,
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> second,
- Collector<Tuple3<Integer, Integer, Integer>> out) throws Exception {
-
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> first,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> second,
+ Collector<Tuple3<Integer, Integer, Integer>> out)
+ {
int sum = 0;
int id = 0;
- while(first.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = first.next();
+ for (Tuple5<Integer, Long, Integer, String, Long> element : first) {
sum += element.f2;
id = element.f0;
}
- while(second.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> element = second.next();
+ for (Tuple5<Integer, Long, Integer, String, Long> element : second) {
sum += element.f2;
id = element.f0;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index 7376e86..bd10c5e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -422,16 +422,13 @@ public class GroupReduceITCase extends JavaProgramTestBase {
public static class Tuple3GroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>> {
private static final long serialVersionUID = 1L;
-
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple2<Integer, Long>> out) throws Exception {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, Long>> out) {
int i = 0;
long l = 0l;
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for (Tuple3<Integer, Long, String> t : values) {
i += t.f0;
l = t.f1;
}
@@ -446,24 +443,22 @@ public class GroupReduceITCase extends JavaProgramTestBase {
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple3<Integer, Long, String>> out) throws Exception {
-
- Tuple3<Integer, Long, String> t = values.next();
-
- int sum = t.f0;
- long key = t.f1;
- String concat = t.f2;
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
+ int sum = 0;
+ long key = 0;
+ StringBuilder concat = new StringBuilder();
- while(values.hasNext()) {
- t = values.next();
-
- sum += t.f0;
- concat += "-"+t.f2;
+ for (Tuple3<Integer, Long, String> next : values) {
+ sum += next.f0;
+ key = next.f1;
+ concat.append(next.f2).append("-");
}
- out.collect(new Tuple3<Integer, Long, String>(sum, key, concat));
+ if (concat.length() > 0) {
+ concat.setLength(concat.length() - 1);
+ }
+ out.collect(new Tuple3<Integer, Long, String>(sum, key, concat.toString()));
}
}
@@ -472,16 +467,14 @@ public class GroupReduceITCase extends JavaProgramTestBase {
@Override
public void reduce(
- Iterator<Tuple5<Integer, Long, Integer, String, Long>> values,
+ Iterable<Tuple5<Integer, Long, Integer, String, Long>> values,
Collector<Tuple5<Integer, Long, Integer, String, Long>> out)
- throws Exception {
-
+ {
int i = 0;
long l = 0l;
long l2 = 0l;
- while(values.hasNext()) {
- Tuple5<Integer, Long, Integer, String, Long> t = values.next();
+ for ( Tuple5<Integer, Long, Integer, String, Long> t : values ) {
i = t.f0;
l += t.f1;
l2 = t.f4;
@@ -496,20 +489,19 @@ public class GroupReduceITCase extends JavaProgramTestBase {
@Override
- public void reduce(Iterator<CustomType> values,
- Collector<CustomType> out) throws Exception {
+ public void reduce(Iterable<CustomType> values, Collector<CustomType> out) {
+ final Iterator<CustomType> iter = values.iterator();
CustomType o = new CustomType();
- CustomType c = values.next();
+ CustomType c = iter.next();
o.myString = "Hello!";
o.myInt = c.myInt;
o.myLong = c.myLong;
- while(values.hasNext()) {
- c = values.next();
- o.myLong += c.myLong;
-
+ while (iter.hasNext()) {
+ CustomType next = iter.next();
+ o.myLong += next.myLong;
}
out.collect(o);
@@ -522,11 +514,9 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple3<Integer, Long, String>> out) throws Exception {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
if(t.f0 < 4) {
t.f2 = "Hi!";
@@ -544,14 +534,12 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple3<Integer, Long, String>> out) throws Exception {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
int i = 0;
long l = 0l;
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
i += t.f0;
l += t.f1;
}
@@ -564,21 +552,13 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<CustomType> values,
- Collector<CustomType> out) throws Exception {
+ public void reduce(Iterable<CustomType> values, Collector<CustomType> out) {
- CustomType o = new CustomType();
- CustomType c = values.next();
-
- o.myString = "Hello!";
- o.myInt = c.myInt;
- o.myLong = c.myLong;
-
+ CustomType o = new CustomType(0, 0, "Hello!");
- while(values.hasNext()) {
- c = values.next();
- o.myInt += c.myInt;
- o.myLong += c.myLong;
+ for (CustomType next : values) {
+ o.myInt += next.myInt;
+ o.myLong += next.myLong;
}
out.collect(o);
@@ -602,14 +582,12 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple3<Integer, Long, String>> out) throws Exception {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
int i = 0;
long l = 0l;
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
i += t.f0;
l = t.f1;
}
@@ -624,12 +602,11 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void combine(Iterator<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) throws Exception {
+ public void combine(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
Tuple3<Integer, Long, String> o = new Tuple3<Integer, Long, String>(0, 0l, "");
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
o.f0 += t.f0;
o.f1 = t.f1;
o.f2 = "test"+o.f1;
@@ -639,14 +616,12 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values,
- Collector<Tuple2<Integer, String>> out) throws Exception {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, String>> out) {
int i = 0;
String s = "";
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
i += t.f0;
s = t.f2;
}
@@ -661,12 +636,11 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void combine(Iterator<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
+ public void combine(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {
Tuple3<Integer, Long, String> o = new Tuple3<Integer, Long, String>(0, 0l, "");
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
o.f0 += t.f0;
o.f1 += t.f1;
o.f2 += "test";
@@ -676,13 +650,12 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
@Override
- public void reduce(Iterator<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, String>> out) {
+ public void reduce(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple2<Integer, String>> out) {
int i = 0;
String s = "";
- while(values.hasNext()) {
- Tuple3<Integer, Long, String> t = values.next();
+ for ( Tuple3<Integer, Long, String> t : values ) {
i += t.f0 + t.f1;
s += t.f2;
}
@@ -697,12 +670,11 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static final long serialVersionUID = 1L;
@Override
- public void combine(Iterator<CustomType> values, Collector<CustomType> out) throws Exception {
+ public void combine(Iterable<CustomType> values, Collector<CustomType> out) throws Exception {
CustomType o = new CustomType();
- while(values.hasNext()) {
- CustomType c = values.next();
+ for ( CustomType c : values ) {
o.myInt = c.myInt;
o.myLong += c.myLong;
o.myString = "test"+c.myInt;
@@ -712,13 +684,11 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
@Override
- public void reduce(Iterator<CustomType> values,
- Collector<CustomType> out) throws Exception {
+ public void reduce(Iterable<CustomType> values, Collector<CustomType> out) {
CustomType o = new CustomType(0, 0, "");
- while(values.hasNext()) {
- CustomType c = values.next();
+ for ( CustomType c : values) {
o.myInt = c.myInt;
o.myLong += c.myLong;
o.myString = c.myString;
@@ -733,6 +703,5 @@ public class GroupReduceITCase extends JavaProgramTestBase {
@Override
public T map(T value) { return value; }
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
index 5a895d3..865c550 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CoGroupITCase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.operators;
import java.io.FileNotFoundException;
@@ -26,8 +25,6 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.record.functions.CoGroupFunction;
import org.apache.flink.api.java.record.io.DelimitedInputFormat;
@@ -46,13 +43,9 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-/**
- */
@RunWith(Parameterized.class)
public class CoGroupITCase extends RecordAPITestBase {
- private static final Log LOG = LogFactory.getLog(CoGroupITCase.class);
-
String leftInPath = null;
String rightInPath = null;
String resultPath = null;
@@ -89,8 +82,6 @@ public class CoGroupITCase extends RecordAPITestBase {
target.setField(0, keyString);
target.setField(1, valueString);
- LOG.debug("Read in: [" + keyString.getValue() + "," + valueString.getValue() + "]");
-
return target;
}
@@ -112,9 +103,6 @@ public class CoGroupITCase extends RecordAPITestBase {
this.buffer.append('\n');
byte[] bytes = this.buffer.toString().getBytes();
-
- LOG.debug("Writing out: [" + keyString.toString() + "," + valueInteger.getValue() + "]");
-
this.stream.write(bytes);
}
}
@@ -124,33 +112,28 @@ public class CoGroupITCase extends RecordAPITestBase {
private StringValue keyString = new StringValue();
private StringValue valueString = new StringValue();
- private Record record = new Record();
@Override
public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) {
- // TODO Auto-generated method stub
+ Record record = null;
int sum = 0;
- LOG.debug("Start iterating over input1");
+
while (records1.hasNext()) {
record = records1.next();
keyString = record.getField(0, keyString);
valueString = record.getField(1, valueString);
sum += Integer.parseInt(valueString.getValue());
-
- LOG.debug("Processed: [" + keyString.getValue() + "," + valueString.getValue() + "]");
}
- LOG.debug("Start iterating over input2");
+
+
while (records2.hasNext()) {
record = records2.next();
keyString = record.getField(0, keyString);
valueString = record.getField(1, valueString);
sum -= Integer.parseInt(valueString.getValue());
-
- LOG.debug("Processed: [" + keyString.getValue() + "," + valueString.getValue() + "]");
}
record.setField(1, new IntValue(sum));
- LOG.debug("Finished");
out.collect(record);
}
@@ -197,9 +180,9 @@ public class CoGroupITCase extends RecordAPITestBase {
LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
- String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE};
+ String[] localStrategies = { PactCompiler.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE };
- String[] shipStrategies = { PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH, };
+ String[] shipStrategies = { PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH };
for (String localStrategy : localStrategies) {
for (String shipStrategy : shipStrategies) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
index b06cf47..2711417 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/ReduceITCase.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.test.operators;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.io.DelimitedInputFormat;
@@ -53,8 +50,6 @@ import java.util.LinkedList;
@RunWith(Parameterized.class)
public class ReduceITCase extends RecordAPITestBase {
-
- private static final Log LOG = LogFactory.getLog(ReduceITCase.class);
String inPath = null;
String resultPath = null;
@@ -83,17 +78,14 @@ public class ReduceITCase extends RecordAPITestBase {
private StringValue combineValue = new StringValue();
@Override
- public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
-
+ public void combine(Iterator<Record> records, Collector<Record> out) {
+ Record record = null;
int sum = 0;
- Record record = new Record();
+
while (records.hasNext()) {
record = records.next();
combineValue = record.getField(1, combineValue);
sum += Integer.parseInt(combineValue.toString());
-
- LOG.debug("Processed: [" + record.getField(0, StringValue.class).toString() +
- "," + combineValue.toString() + "]");
}
combineValue.setValue(sum + "");
record.setField(1, combineValue);
@@ -101,17 +93,14 @@ public class ReduceITCase extends RecordAPITestBase {
}
@Override
- public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
-
+ public void reduce(Iterator<Record> records, Collector<Record> out) {
+ Record record = null;
int sum = 0;
- Record record = new Record();
+
while (records.hasNext()) {
record = records.next();
reduceValue = record.getField(1, reduceValue);
sum += Integer.parseInt(reduceValue.toString());
-
- LOG.debug("Processed: [" + record.getField(0, StringValue.class).toString() +
- "," + reduceValue.toString() + "]");
}
record.setField(1, new IntValue(sum));
out.collect(record);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-tests/src/test/java/org/apache/flink/test/recordJobTests/GroupOrderReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recordJobTests/GroupOrderReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recordJobTests/GroupOrderReduceITCase.java
index 60b7512..0b84309 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recordJobTests/GroupOrderReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recordJobTests/GroupOrderReduceITCase.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.test.recordJobTests;
import java.io.Serializable;
[16/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
index a99ffc8..e57d20c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CoGroupTaskTest.java
@@ -24,12 +24,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
-import org.apache.flink.api.java.record.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
-import org.apache.flink.runtime.operators.CoGroupDriver;
-import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.CoGroupTaskExternalITCase.MockCoGroupStub;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -42,7 +39,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class CoGroupTaskTest extends DriverTestBase<GenericCoGrouper<Record, Record, Record>>
+public class CoGroupTaskTest extends DriverTestBase<CoGroupFunction<Record, Record, Record>>
{
private static final long SORT_MEM = 3*1024*1024;
@@ -403,7 +400,7 @@ public class CoGroupTaskTest extends DriverTestBase<GenericCoGrouper<Record, Rec
Assert.assertTrue("Test threw an exception even though it was properly canceled.", success.get());
}
- public static class MockFailingCoGroupStub extends CoGroupFunction {
+ public static class MockFailingCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@@ -443,7 +440,7 @@ public class CoGroupTaskTest extends DriverTestBase<GenericCoGrouper<Record, Rec
}
- public static final class MockDelayingCoGroupStub extends CoGroupFunction {
+ public static final class MockDelayingCoGroupStub extends org.apache.flink.api.java.record.functions.CoGroupFunction {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
index 694ae48..885a509 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskExternalITCase.java
@@ -24,10 +24,8 @@ import java.util.HashMap;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.GroupReduceCombineDriver;
import org.apache.flink.runtime.operators.CombineTaskTest.MockCombiningReduceStub;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
@@ -37,7 +35,7 @@ import org.apache.flink.types.Record;
import org.junit.Test;
-public class CombineTaskExternalITCase extends DriverTestBase<GenericGroupReduce<Record, ?>> {
+public class CombineTaskExternalITCase extends DriverTestBase<RichGroupReduceFunction<Record, ?>> {
private static final long COMBINE_MEM = 3 * 1024 * 1024;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
index f42c443..99440aa 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CombineTaskTest.java
@@ -25,12 +25,10 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.GroupReduceCombineDriver;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -43,7 +41,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class CombineTaskTest extends DriverTestBase<GenericGroupReduce<Record, ?>>
+public class CombineTaskTest extends DriverTestBase<RichGroupReduceFunction<Record, ?>>
{
private static final long COMBINE_MEM = 3 * 1024 * 1024;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
index 79d47fe..ced86d1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskExternalITCase.java
@@ -21,9 +21,7 @@ package org.apache.flink.runtime.operators;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.runtime.operators.CrossDriver;
-import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.runtime.operators.CrossTaskTest.MockCrossStub;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
@@ -31,7 +29,7 @@ import org.apache.flink.types.Record;
import org.junit.Test;
-public class CrossTaskExternalITCase extends DriverTestBase<GenericCrosser<Record, Record, Record>>
+public class CrossTaskExternalITCase extends DriverTestBase<CrossFunction<Record, Record, Record>>
{
private static final long CROSS_MEM = 1024 * 1024;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
index 01015d3..ee51ad0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/CrossTaskTest.java
@@ -23,20 +23,16 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.api.java.record.functions.CrossFunction;
-import org.apache.flink.runtime.operators.CrossDriver;
-import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
import org.apache.flink.runtime.operators.testutils.TaskCancelThread;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
import org.junit.Test;
-public class CrossTaskTest extends DriverTestBase<GenericCrosser<Record, Record, Record>>
+public class CrossTaskTest extends DriverTestBase<CrossFunction<Record, Record, Record>>
{
private static final long CROSS_MEM = 1024 * 1024;
@@ -583,26 +579,26 @@ public class CrossTaskTest extends DriverTestBase<GenericCrosser<Record, Record,
Assert.assertTrue("Exception was thrown despite proper canceling.", success.get());
}
- public static final class MockCrossStub extends CrossFunction {
+ public static final class MockCrossStub extends org.apache.flink.api.java.record.functions.CrossFunction {
private static final long serialVersionUID = 1L;
-
+
@Override
- public void cross(Record record1, Record record2, Collector<Record> out) {
- out.collect(record1);
+ public Record cross(Record record1, Record record2) throws Exception {
+ return record1;
}
}
- public static final class MockFailingCrossStub extends CrossFunction {
+ public static final class MockFailingCrossStub extends org.apache.flink.api.java.record.functions.CrossFunction {
private static final long serialVersionUID = 1L;
private int cnt = 0;
@Override
- public void cross(Record record1, Record record2, Collector<Record> out) {
+ public Record cross(Record record1, Record record2) {
if (++this.cnt >= 10) {
throw new ExpectedTestException();
}
- out.collect(record1);
+ return record1;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
index 2115bc8..25998f3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskExternalITCase.java
@@ -21,12 +21,10 @@ package org.apache.flink.runtime.operators;
import org.junit.Assert;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.java.record.functions.JoinFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.MatchDriver;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
import org.apache.flink.types.IntValue;
@@ -35,7 +33,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class MatchTaskExternalITCase extends DriverTestBase<GenericJoiner<Record, Record, Record>>
+public class MatchTaskExternalITCase extends DriverTestBase<FlatJoinFunction<Record, Record, Record>>
{
private static final long HASH_MEM = 4*1024*1024;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskTest.java
index 6eea546..9b7c582 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/MatchTaskTest.java
@@ -23,12 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.java.record.functions.JoinFunction;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordPairComparatorFactory;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.MatchDriver;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
@@ -42,7 +40,7 @@ import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
-public class MatchTaskTest extends DriverTestBase<GenericJoiner<Record, Record, Record>>
+public class MatchTaskTest extends DriverTestBase<FlatJoinFunction<Record, Record, Record>>
{
private static final long HASH_MEM = 6*1024*1024;
@@ -951,11 +949,10 @@ public class MatchTaskTest extends DriverTestBase<GenericJoiner<Record, Record,
private int cnt = 0;
@Override
- public void join(Record record1, Record record2, Collector<Record> out) {
+ public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
if (++this.cnt >= 10) {
throw new ExpectedTestException();
}
-
out.collect(record1);
}
}
@@ -964,10 +961,11 @@ public class MatchTaskTest extends DriverTestBase<GenericJoiner<Record, Record,
private static final long serialVersionUID = 1L;
@Override
- public void join(Record record1, Record record2, Collector<Record> out) {
+ public void join(Record record1, Record record2, Collector<Record> out) throws Exception {
try {
Thread.sleep(100);
- } catch (InterruptedException e) { }
+ } catch (InterruptedException e) {
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
index bf51f41..ebee0b8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskExternalITCase.java
@@ -27,13 +27,11 @@ import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.GroupReduceDriver;
import org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
import org.apache.flink.runtime.operators.testutils.UniformRecordGenerator;
@@ -43,7 +41,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class ReduceTaskExternalITCase extends DriverTestBase<GenericGroupReduce<Record, Record>>
+public class ReduceTaskExternalITCase extends DriverTestBase<RichGroupReduceFunction<Record, Record>>
{
private static final Log LOG = LogFactory.getLog(ReduceTaskExternalITCase.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
index 0d4e370..b367a7e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/ReduceTaskTest.java
@@ -28,13 +28,11 @@ import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.record.functions.ReduceFunction;
import org.apache.flink.api.java.record.operators.ReduceOperator.Combinable;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
import org.apache.flink.api.java.typeutils.runtime.record.RecordSerializerFactory;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.GroupReduceDriver;
import org.apache.flink.runtime.operators.sort.CombiningUnilateralSortMerger;
import org.apache.flink.runtime.operators.testutils.DelayingInfinitiveInputIterator;
import org.apache.flink.runtime.operators.testutils.DriverTestBase;
@@ -48,7 +46,7 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.junit.Test;
-public class ReduceTaskTest extends DriverTestBase<GenericGroupReduce<Record, Record>>
+public class ReduceTaskTest extends DriverTestBase<RichGroupReduceFunction<Record, Record>>
{
private static final Log LOG = LogFactory.getLog(ReduceTaskTest.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
index 8c8715e..d603dec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllGroupReduceDriverTest.java
@@ -23,8 +23,8 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.runtime.operators.AllGroupReduceDriver;
@@ -46,8 +46,8 @@ public class AllGroupReduceDriverTest {
@Test
public void testAllReduceDriverImmutableEmpty() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -72,8 +72,8 @@ public class AllGroupReduceDriverTest {
@Test
public void testAllReduceDriverImmutable() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -112,8 +112,8 @@ public class AllGroupReduceDriverTest {
@Test
public void testAllReduceDriverMutable() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -152,7 +152,7 @@ public class AllGroupReduceDriverTest {
// Test UDFs
// --------------------------------------------------------------------------------------------
- public static final class ConcatSumReducer extends GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
+ public static final class ConcatSumReducer extends RichGroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
@Override
public void reduce(Iterator<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
@@ -169,7 +169,7 @@ public class AllGroupReduceDriverTest {
}
}
- public static final class ConcatSumMutableReducer extends GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumMutableReducer extends RichGroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
@Override
public void reduce(Iterator<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllReduceDriverTest.java
index de01d74..b124e51 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/AllReduceDriverTest.java
@@ -22,8 +22,8 @@ package org.apache.flink.runtime.operators.drivers;
import java.util.Arrays;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericReduce;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.runtime.operators.AllReduceDriver;
@@ -44,8 +44,8 @@ public class AllReduceDriverTest {
@Test
public void testAllReduceDriverImmutableEmpty() {
try {
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -71,8 +71,8 @@ public class AllReduceDriverTest {
public void testAllReduceDriverImmutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -103,8 +103,8 @@ public class AllReduceDriverTest {
}
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -145,8 +145,8 @@ public class AllReduceDriverTest {
public void testAllReduceDriverMutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -176,8 +176,8 @@ public class AllReduceDriverTest {
Assert.assertEquals(78, res.f1.getValue());
}
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0));
@@ -217,7 +217,7 @@ public class AllReduceDriverTest {
// Test UDFs
// --------------------------------------------------------------------------------------------
- public static final class ConcatSumFirstReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumFirstReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -227,7 +227,7 @@ public class AllReduceDriverTest {
}
}
- public static final class ConcatSumSecondReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumSecondReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -237,7 +237,7 @@ public class AllReduceDriverTest {
}
}
- public static final class ConcatSumFirstMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumFirstMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
@@ -247,7 +247,7 @@ public class AllReduceDriverTest {
}
}
- public static final class ConcatSumSecondMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumSecondMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
index e5a01d6..a325e33 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
@@ -22,9 +22,9 @@ package org.apache.flink.runtime.operators.drivers;
import java.util.Iterator;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -46,8 +46,8 @@ public class GroupReduceDriverTest {
@Test
public void testAllReduceDriverImmutableEmpty() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
@@ -74,8 +74,8 @@ public class GroupReduceDriverTest {
@Test
public void testAllReduceDriverImmutable() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
@@ -110,8 +110,8 @@ public class GroupReduceDriverTest {
@Test
public void testAllReduceDriverMutable() {
try {
- TestTaskContext<GenericGroupReduce<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericGroupReduce<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
@@ -149,7 +149,7 @@ public class GroupReduceDriverTest {
// Test UDFs
// --------------------------------------------------------------------------------------------
- public static final class ConcatSumReducer extends GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
+ public static final class ConcatSumReducer extends RichGroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>> {
@Override
public void reduce(Iterator<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) {
@@ -166,7 +166,7 @@ public class GroupReduceDriverTest {
}
}
- public static final class ConcatSumMutableReducer extends GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumMutableReducer extends RichGroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> {
@Override
public void reduce(Iterator<Tuple2<StringValue, IntValue>> values, Collector<Tuple2<StringValue, IntValue>> out) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
index 734e119..ae9c294 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
@@ -22,9 +22,9 @@ package org.apache.flink.runtime.operators.drivers;
import java.util.Collections;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -44,8 +44,8 @@ public class ReduceCombineDriverTest {
@Test
public void testImmutableEmpty() {
try {
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
@@ -81,8 +81,8 @@ public class ReduceCombineDriverTest {
public void testReduceDriverImmutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
@@ -112,8 +112,8 @@ public class ReduceCombineDriverTest {
}
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
@@ -153,8 +153,8 @@ public class ReduceCombineDriverTest {
public void testReduceDriverMutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(1024 * 1024);
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
@@ -181,8 +181,8 @@ public class ReduceCombineDriverTest {
DriverTestData.compareTupleArrays(expected, res);
}
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(1024 * 1024);
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(1024 * 1024);
context.getTaskConfig().setRelativeMemoryDriver(0.5);
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
@@ -220,7 +220,7 @@ public class ReduceCombineDriverTest {
// Test UDFs
// --------------------------------------------------------------------------------------------
- public static final class ConcatSumFirstReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumFirstReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -230,7 +230,7 @@ public class ReduceCombineDriverTest {
}
}
- public static final class ConcatSumSecondReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumSecondReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -240,7 +240,7 @@ public class ReduceCombineDriverTest {
}
}
- public static final class ConcatSumFirstMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumFirstMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
@@ -250,7 +250,7 @@ public class ReduceCombineDriverTest {
}
}
- public static final class ConcatSumSecondMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumSecondMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
index e8370cf..28217b4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
@@ -21,9 +21,9 @@ package org.apache.flink.runtime.operators.drivers;
import java.util.List;
-import org.apache.flink.api.common.functions.GenericReduce;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -43,8 +43,8 @@ public class ReduceDriverTest {
@Test
public void testReduceDriverImmutableEmpty() {
try {
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
@@ -76,8 +76,8 @@ public class ReduceDriverTest {
public void testReduceDriverImmutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
@@ -104,8 +104,8 @@ public class ReduceDriverTest {
}
{
- TestTaskContext<GenericReduce<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
- new TestTaskContext<GenericReduce<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
+ TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>();
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
@@ -142,8 +142,8 @@ public class ReduceDriverTest {
public void testReduceDriverMutable() {
try {
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
@@ -169,8 +169,8 @@ public class ReduceDriverTest {
DriverTestData.compareTupleArrays(expected, res);
}
{
- TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
- new TestTaskContext<GenericReduce<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
+ TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context =
+ new TestTaskContext<ReduceFunction<Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
@@ -206,7 +206,7 @@ public class ReduceDriverTest {
// Test UDFs
// --------------------------------------------------------------------------------------------
- public static final class ConcatSumFirstReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumFirstReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -216,7 +216,7 @@ public class ReduceDriverTest {
}
}
- public static final class ConcatSumSecondReducer extends ReduceFunction<Tuple2<String, Integer>> {
+ public static final class ConcatSumSecondReducer extends RichReduceFunction<Tuple2<String, Integer>> {
@Override
public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) {
@@ -226,7 +226,7 @@ public class ReduceDriverTest {
}
}
- public static final class ConcatSumFirstMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumFirstMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
@@ -236,7 +236,7 @@ public class ReduceDriverTest {
}
}
- public static final class ConcatSumSecondMutableReducer extends ReduceFunction<Tuple2<StringValue, IntValue>> {
+ public static final class ConcatSumSecondMutableReducer extends RichReduceFunction<Tuple2<StringValue, IntValue>> {
@Override
public Tuple2<StringValue, IntValue> reduce(Tuple2<StringValue, IntValue> value1, Tuple2<StringValue, IntValue> value2) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashMatchIteratorITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashMatchIteratorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashMatchIteratorITCase.java
index 303d921..ac8cc78 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashMatchIteratorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/hash/HashMatchIteratorITCase.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -39,8 +39,6 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager;
import org.apache.flink.runtime.memorymanager.MemoryManager;
-import org.apache.flink.runtime.operators.hash.BuildFirstHashMatchIterator;
-import org.apache.flink.runtime.operators.hash.BuildSecondHashMatchIterator;
import org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector;
import org.apache.flink.runtime.operators.testutils.DummyInvokable;
import org.apache.flink.runtime.operators.testutils.TestData;
@@ -414,7 +412,7 @@ public class HashMatchIteratorITCase {
collectIntPairData(input1),
collectRecordData(input2));
- final GenericJoiner<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
+ final FlatJoinFunction<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
final Collector<Record> collector = new DiscardingOutputCollector<Record>();
// reset the generators
@@ -425,7 +423,7 @@ public class HashMatchIteratorITCase {
// compare with iterator values
BuildSecondHashMatchIterator<IntPair, Record, Record> iterator =
new BuildSecondHashMatchIterator<IntPair, Record, Record>(
- input1, input2, this.pairSerializer, this.pairComparator,
+ input1, input2, this.pairSerializer, this.pairComparator,
this.recordSerializer, this.record2Comparator, this.pairRecordPairComparator,
this.memoryManager, this.ioManager, this.parentTask, 1.0);
@@ -461,7 +459,7 @@ public class HashMatchIteratorITCase {
collectIntPairData(input1),
collectRecordData(input2));
- final GenericJoiner<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
+ final FlatJoinFunction<IntPair, Record, Record> matcher = new RecordIntPairMatchRemovingMatcher(expectedMatchesMap);
final Collector<Record> collector = new DiscardingOutputCollector<Record>();
// reset the generators
@@ -675,7 +673,7 @@ public class HashMatchIteratorITCase {
}
@Override
- public void join(Record rec1, Record rec2, Collector<Record> out)
+ public void join(Record rec1, Record rec2, Collector<Record> out) throws Exception
{
TestData.Key key = rec1.getField(0, TestData.Key.class);
TestData.Value value1 = rec1.getField(1, TestData.Value.class);
@@ -695,7 +693,7 @@ public class HashMatchIteratorITCase {
}
}
- static final class RecordIntPairMatchRemovingMatcher extends AbstractFunction implements GenericJoiner<IntPair, Record, Record>
+ static final class RecordIntPairMatchRemovingMatcher extends AbstractRichFunction implements FlatJoinFunction<IntPair, Record, Record>
{
private final Map<TestData.Key, Collection<RecordIntPairMatch>> toRemoveFrom;
@@ -704,7 +702,7 @@ public class HashMatchIteratorITCase {
}
@Override
- public void join(IntPair rec1, Record rec2, Collector<Record> out)
+ public void join(IntPair rec1, Record rec2, Collector<Record> out) throws Exception
{
final int k = rec1.getKey();
final int v = rec1.getValue();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeMatchIteratorITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeMatchIteratorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeMatchIteratorITCase.java
index e2de4cb..93e52f6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeMatchIteratorITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/SortMergeMatchIteratorITCase.java
@@ -37,6 +37,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager;
import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
import org.apache.flink.runtime.operators.sort.MergeIterator;
import org.apache.flink.runtime.operators.sort.MergeMatchIterator;
import org.apache.flink.runtime.operators.testutils.DiscardingOutputCollector;
@@ -353,8 +354,7 @@ public class SortMergeMatchIteratorITCase {
}
@Override
- public void join(Record rec1, Record rec2, Collector<Record> out)
- {
+ public void join(Record rec1, Record rec2, Collector<Record> out) throws Exception {
TestData.Key key = rec1.getField(0, TestData.Key.class);
TestData.Value value1 = rec1.getField(1, TestData.Value.class);
TestData.Value value2 = rec2.getField(1, TestData.Value.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
index 9c177a6..e6661ec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.junit.Assert;
import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.typeutils.runtime.record.RecordComparator;
@@ -173,7 +175,7 @@ public class DriverTestBase<S extends Function> implements PactTaskContext<S, Re
// open stub implementation
try {
- this.stub.open(getTaskConfig().getStubParameters());
+ FunctionUtils.openFunction(this.stub, getTaskConfig().getStubParameters());
stubOpen = true;
}
catch (Throwable t) {
@@ -185,7 +187,7 @@ public class DriverTestBase<S extends Function> implements PactTaskContext<S, Re
// close. We close here such that a regular close throwing an exception marks a task as failed.
if (this.running) {
- this.stub.close();
+ FunctionUtils.closeFunction (this.stub);
stubOpen = false;
}
@@ -195,7 +197,7 @@ public class DriverTestBase<S extends Function> implements PactTaskContext<S, Re
// close the input, but do not report any exceptions, since we already have another root cause
if (stubOpen) {
try {
- this.stub.close();
+ FunctionUtils.closeFunction(this.stub);
}
catch (Throwable t) {}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
index faa87b5..779640d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/TaskTestBase.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.testutils;
import java.util.List;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.java.record.io.DelimitedInputFormat;
@@ -80,10 +80,10 @@ public abstract class TaskTestBase {
return this.mockEnv.getTaskConfiguration();
}
- public void registerTask(AbstractInvokable task, @SuppressWarnings("rawtypes") Class<? extends PactDriver> driver, Class<? extends Function> stubClass) {
+ public void registerTask(AbstractInvokable task, @SuppressWarnings("rawtypes") Class<? extends PactDriver> driver, Class<? extends RichFunction> stubClass) {
final TaskConfig config = new TaskConfig(this.mockEnv.getTaskConfiguration());
config.setDriver(driver);
- config.setStubWrapper(new UserCodeClassWrapper<Function>(stubClass));
+ config.setStubWrapper(new UserCodeClassWrapper<RichFunction>(stubClass));
task.setEnvironment(this.mockEnv);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/HashVsSortMiniBenchmark.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/HashVsSortMiniBenchmark.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/HashVsSortMiniBenchmark.java
index 92b6c44..8b891a9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/HashVsSortMiniBenchmark.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/HashVsSortMiniBenchmark.java
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager;
import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
import org.apache.flink.runtime.operators.hash.BuildFirstHashMatchIterator;
import org.apache.flink.runtime.operators.hash.BuildSecondHashMatchIterator;
import org.apache.flink.runtime.operators.sort.MergeMatchIterator;
@@ -250,6 +251,7 @@ public class HashVsSortMiniBenchmark {
private static final long serialVersionUID = 1L;
@Override
- public void join(Record rec1, Record rec2, Collector<Record> out) {}
+ public void join(Record rec1, Record rec2, Collector<Record> out) throws Exception {
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/CrossFunction.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/CrossFunction.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/CrossFunction.scala
index 6dfd1e3..c292100 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/CrossFunction.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/functions/CrossFunction.scala
@@ -45,7 +45,7 @@ abstract class CrossFunctionBase[LeftIn: UDT, RightIn: UDT, Out: UDT] extends JC
}
abstract class CrossFunction[LeftIn: UDT, RightIn: UDT, Out: UDT] extends CrossFunctionBase[LeftIn, RightIn, Out] with Function2[LeftIn, RightIn, Out] {
- override def cross(leftRecord: Record, rightRecord: Record, out: Collector[Record]) = {
+ override def cross(leftRecord: Record, rightRecord: Record) : Record = {
val left = leftDeserializer.deserializeRecyclingOn(leftRecord)
val right = rightDeserializer.deserializeRecyclingOn(rightRecord)
val output = apply(left, right)
@@ -59,31 +59,7 @@ abstract class CrossFunction[LeftIn: UDT, RightIn: UDT, Out: UDT] extends CrossF
leftRecord.copyFrom(leftRecord, leftForwardFrom, leftForwardTo)
serializer.serialize(output, leftRecord)
- out.collect(leftRecord)
- }
-}
-
-abstract class FlatCrossFunction[LeftIn: UDT, RightIn: UDT, Out: UDT] extends CrossFunctionBase[LeftIn, RightIn, Out] with Function2[LeftIn, RightIn, Iterator[Out]] {
- override def cross(leftRecord: Record, rightRecord: Record, out: Collector[Record]) = {
- val left = leftDeserializer.deserializeRecyclingOn(leftRecord)
- val right = rightDeserializer.deserializeRecyclingOn(rightRecord)
- val output = apply(left, right)
-
- if (output.nonEmpty) {
-
- leftRecord.setNumFields(outputLength)
-
- for (field <- leftDiscard)
- leftRecord.setNull(field)
-
- leftRecord.copyFrom(rightRecord, rightForwardFrom, rightForwardTo)
- leftRecord.copyFrom(leftRecord, leftForwardFrom, leftForwardTo)
-
- for (item <- output) {
- serializer.serialize(item, leftRecord)
- out.collect(leftRecord)
- }
- }
+ leftRecord
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CrossOperator.scala
index 17df2c3..aa04093 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CrossOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/CrossOperator.scala
@@ -27,7 +27,7 @@ import java.util.{ Iterator => JIterator }
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.analysis._
-import org.apache.flink.api.scala.functions.{CrossFunctionBase, CrossFunction, FlatCrossFunction}
+import org.apache.flink.api.scala.functions.{CrossFunctionBase, CrossFunction}//, FlatCrossFunction}
import org.apache.flink.api.scala.codegen.{MacroContextHolder, Util}
import org.apache.flink.api.scala.functions.DeserializingIterator
import org.apache.flink.api.scala.DataSet
@@ -42,7 +42,7 @@ import org.apache.flink.configuration.Configuration
class CrossDataSet[LeftIn, RightIn](val leftInput: DataSet[LeftIn], val rightInput: DataSet[RightIn]) {
def map[Out](fun: (LeftIn, RightIn) => Out): DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out] = macro CrossMacros.map[LeftIn, RightIn, Out]
- def flatMap[Out](fun: (LeftIn, RightIn) => Iterator[Out]): DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out] = macro CrossMacros.flatMap[LeftIn, RightIn, Out]
+ //def flatMap[Out](fun: (LeftIn, RightIn) => Iterator[Out]): DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out] = macro CrossMacros.flatMap[LeftIn, RightIn, Out]
def filter(fun: (LeftIn, RightIn) => Boolean): DataSet[(LeftIn, RightIn)] with TwoInputHintable[LeftIn, RightIn, (LeftIn, RightIn)] = macro CrossMacros.filter[LeftIn, RightIn]
}
@@ -65,7 +65,7 @@ object CrossMacros {
implicit val rightInputUDT: UDT[RightIn] = c.Expr[UDT[RightIn]](createUdtRightIn).splice
implicit val outputUDT: UDT[Out] = c.Expr[UDT[Out]](createUdtOut).splice
new CrossFunctionBase[LeftIn, RightIn, Out] {
- override def cross(leftRecord: Record, rightRecord: Record, out: Collector[Record]) = {
+ override def cross(leftRecord: Record, rightRecord: Record) : Record = {
val left = leftDeserializer.deserializeRecyclingOn(leftRecord)
val right = rightDeserializer.deserializeRecyclingOn(rightRecord)
val output = fun.splice.apply(left, right)
@@ -79,7 +79,7 @@ object CrossMacros {
leftRecord.copyFrom(leftRecord, leftForwardFrom, leftForwardTo)
serializer.serialize(output, leftRecord)
- out.collect(leftRecord)
+ leftRecord
}
}
}
@@ -106,67 +106,6 @@ object CrossMacros {
return result
}
- def flatMap[LeftIn: c.WeakTypeTag, RightIn: c.WeakTypeTag, Out: c.WeakTypeTag](c: Context { type PrefixType = CrossDataSet[LeftIn, RightIn] })(fun: c.Expr[(LeftIn, RightIn) => Iterator[Out]]): c.Expr[DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out]] = {
- import c.universe._
-
- val slave = MacroContextHolder.newMacroHelper(c)
-
- val (udtLeftIn, createUdtLeftIn) = slave.mkUdtClass[LeftIn]
- val (udtRightIn, createUdtRightIn) = slave.mkUdtClass[RightIn]
- val (udtOut, createUdtOut) = slave.mkUdtClass[Out]
-
- val stub: c.Expr[CrossFunctionBase[LeftIn, RightIn, Out]] = if (fun.actualType <:< weakTypeOf[CrossFunction[LeftIn, RightIn, Out]])
- reify { fun.splice.asInstanceOf[CrossFunctionBase[LeftIn, RightIn, Out]] }
- else reify {
- implicit val leftInputUDT: UDT[LeftIn] = c.Expr[UDT[LeftIn]](createUdtLeftIn).splice
- implicit val rightInputUDT: UDT[RightIn] = c.Expr[UDT[RightIn]](createUdtRightIn).splice
- implicit val outputUDT: UDT[Out] = c.Expr[UDT[Out]](createUdtOut).splice
- new CrossFunctionBase[LeftIn, RightIn, Out] {
- override def cross(leftRecord: Record, rightRecord: Record, out: Collector[Record]) = {
- val left = leftDeserializer.deserializeRecyclingOn(leftRecord)
- val right = rightDeserializer.deserializeRecyclingOn(rightRecord)
- val output = fun.splice.apply(left, right)
-
- if (output.nonEmpty) {
-
- leftRecord.setNumFields(outputLength)
-
- for (field <- leftDiscard)
- leftRecord.setNull(field)
-
- leftRecord.copyFrom(rightRecord, rightForwardFrom, rightForwardTo)
- leftRecord.copyFrom(leftRecord, leftForwardFrom, leftForwardTo)
-
- for (item <- output) {
- serializer.serialize(item, leftRecord)
- out.collect(leftRecord)
- }
- }
- }
- }
- }
- val contract = reify {
- val helper: CrossDataSet[LeftIn, RightIn] = c.prefix.splice
- val leftInput = helper.leftInput.contract
- val rightInput = helper.rightInput.contract
- val generatedStub = ClosureCleaner.clean(stub.splice)
- val builder = CrossOperator.builder(generatedStub).input1(leftInput).input2(rightInput)
-
- val ret = new CrossOperator(builder) with TwoInputScalaOperator[LeftIn, RightIn, Out] {
- override def getUDF = generatedStub.udf
- override def annotations = Seq(
- Annotations.getConstantFieldsFirst(
- Util.filterNonForwards(getUDF.getLeftForwardIndexArrayFrom, getUDF.getLeftForwardIndexArrayTo)),
- Annotations.getConstantFieldsSecond(
- Util.filterNonForwards(getUDF.getRightForwardIndexArrayFrom, getUDF.getRightForwardIndexArrayTo)))
- }
- new DataSet[Out](ret) with TwoInputHintable[LeftIn, RightIn, Out] {}
- }
-
- val result = c.Expr[DataSet[Out] with TwoInputHintable[LeftIn, RightIn, Out]](Block(List(udtLeftIn, udtRightIn, udtOut), contract.tree))
-
- return result
- }
def filter[LeftIn: c.WeakTypeTag, RightIn: c.WeakTypeTag](c: Context { type PrefixType = CrossDataSet[LeftIn, RightIn] })(fun: c.Expr[(LeftIn, RightIn) => Boolean]): c.Expr[DataSet[(LeftIn, RightIn)] with TwoInputHintable[LeftIn, RightIn, (LeftIn, RightIn)]] = {
import c.universe._
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/IterateOperators.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/IterateOperators.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/IterateOperators.scala
index 759b444..66d94ba 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/IterateOperators.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/operators/IterateOperators.scala
@@ -33,7 +33,7 @@ import org.apache.flink.api.scala.analysis.UDF0
import org.apache.flink.api.scala.analysis.FieldSelector
import org.apache.flink.configuration.Configuration
-import org.apache.flink.api.common.functions.AbstractFunction
+import org.apache.flink.api.common.functions.AbstractRichFunction
import org.apache.flink.api.java.record.operators.BulkIteration
import org.apache.flink.api.common.operators.base.BulkIterationBase
import org.apache.flink.api.java.record.operators.DeltaIteration
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
index f450483..ccdd52e 100644
--- a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
+++ b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/CompilerTestBase.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.io.FileInputFormat.FileBaseStatistics;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
@@ -166,7 +166,7 @@ public abstract class CompilerTestBase {
}
@SuppressWarnings("unchecked")
- public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
+ public <T extends PlanNode> T getNode(String name, Class<? extends RichFunction> stubClass) {
List<PlanNode> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/OperatorResolver.java
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/OperatorResolver.java b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/OperatorResolver.java
index de27390..28bdd01 100644
--- a/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/OperatorResolver.java
+++ b/flink-test-utils/src/main/java/org/apache/flink/test/compiler/util/OperatorResolver.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.java.record.operators.BulkIteration;
import org.apache.flink.api.java.record.operators.DeltaIteration;
@@ -63,7 +63,7 @@ public class OperatorResolver implements Visitor<Operator<?>> {
}
@SuppressWarnings("unchecked")
- public <T extends Operator<?>> T getNode(String name, Class<? extends Function> stubClass) {
+ public <T extends Operator<?>> T getNode(String name, Class<? extends RichFunction> stubClass) {
List<Operator<?>> nodes = this.map.get(name);
if (nodes == null || nodes.isEmpty()) {
throw new RuntimeException("No node found with the given name and stub class.");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index e6ce7c1..950e78e 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -35,6 +35,7 @@ under the License.
<packaging>jar</packaging>
+
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -184,7 +185,6 @@ under the License.
</execution>
</executions>
</plugin>
-
</plugins>
<pluginManagement>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
index 15cfac3..e0a96b3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/compiler/iterations/MultipleJoinsWithSolutionSetCompilerTest.java
@@ -23,9 +23,9 @@ import static org.junit.Assert.fail;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.runtime.operators.DriverStrategy;
@@ -106,7 +106,7 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
return result;
}
- public static final class SummingJoin extends JoinFunction<Tuple2<Long, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
+ public static final class SummingJoin extends RichJoinFunction<Tuple2<Long, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public Tuple2<Long, Double> join(Tuple2<Long, Double> first, Tuple2<Long, Double> second) {
@@ -114,7 +114,7 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
}
}
- public static final class SummingJoinProject extends JoinFunction<Tuple3<Long, Double, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
+ public static final class SummingJoinProject extends RichJoinFunction<Tuple3<Long, Double, Double>, Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public Tuple2<Long, Double> join(Tuple3<Long, Double, Double> first, Tuple2<Long, Double> second) {
@@ -122,7 +122,7 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
}
}
- public static final class Duplicator extends FlatMapFunction<Tuple2<Long, Double>, Tuple2<Long, Double>> {
+ public static final class Duplicator extends RichFlatMapFunction<Tuple2<Long, Double>, Tuple2<Long, Double>> {
@Override
public void flatMap(Tuple2<Long, Double> value, Collector<Tuple2<Long, Double>> out) {
@@ -131,7 +131,7 @@ public class MultipleJoinsWithSolutionSetCompilerTest extends CompilerTestBase {
}
}
- public static final class Expander extends MapFunction<Tuple2<Long, Double>, Tuple3<Long, Double, Double>> {
+ public static final class Expander extends RichMapFunction<Tuple2<Long, Double>, Tuple3<Long, Double, Double>> {
@Override
public Tuple3<Long, Double, Double> map(Tuple2<Long, Double> value) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
index 857f321..35aea4b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/BulkIterationWithAllReducerITCase.java
@@ -24,7 +24,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
@@ -61,7 +61,7 @@ public class BulkIterationWithAllReducerITCase extends JavaProgramTestBase {
}
- public static class PickOneAllReduce extends GroupReduceFunction<Integer, Integer> {
+ public static class PickOneAllReduce extends RichGroupReduceFunction<Integer, Integer> {
private Integer bcValue;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
index 31540dc..49bdb26 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/CoGroupConnectedComponentsSecondITCase.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsFirst;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsSecond;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -90,7 +90,7 @@ public class CoGroupConnectedComponentsSecondITCase extends JavaProgramTestBase
// The test program
// --------------------------------------------------------------------------------------------
- public static final class VertexParser extends MapFunction<String, Long> {
+ public static final class VertexParser extends RichMapFunction<String, Long> {
@Override
public Long map(String value) throws Exception {
@@ -98,7 +98,7 @@ public class CoGroupConnectedComponentsSecondITCase extends JavaProgramTestBase
}
}
- public static final class EdgeParser extends FlatMapFunction<String, Tuple2<Long, Long>> {
+ public static final class EdgeParser extends RichFlatMapFunction<String, Tuple2<Long, Long>> {
@Override
public void flatMap(String value, Collector<Tuple2<Long, Long>> out) throws Exception {
@@ -113,7 +113,7 @@ public class CoGroupConnectedComponentsSecondITCase extends JavaProgramTestBase
@ConstantFieldsFirst("0")
@ConstantFieldsSecond("0")
- public static final class MinIdAndUpdate extends CoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class MinIdAndUpdate extends RichCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
@Override
public void coGroup(Iterator<Tuple2<Long, Long>> candidates, Iterator<Tuple2<Long, Long>> current, Collector<Tuple2<Long, Long>> out) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
index be24662..d2669aa 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/DependencyConnectedComponentsITCase.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.test.util.JavaProgramTestBase;
@@ -153,8 +153,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class FindCandidatesJoin extends JoinFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>, Long> {
+ public static final class FindCandidatesJoin extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Long> {
private static final long serialVersionUID = 1L;
@@ -166,7 +165,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class RemoveDuplicatesReduce extends GroupReduceFunction<Long, Long> {
+ public static final class RemoveDuplicatesReduce extends RichGroupReduceFunction<Long, Long> {
private static final long serialVersionUID = 1L;
@@ -176,8 +175,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class FindCandidatesDependenciesJoin extends JoinFunction
- <Long, Tuple2<Long, Long>,Tuple2<Long, Long>> {
+ public static final class FindCandidatesDependenciesJoin extends RichJoinFunction<Long, Tuple2<Long, Long>,Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
@@ -187,8 +185,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class NeighborWithComponentIDJoin extends JoinFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class NeighborWithComponentIDJoin extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
@@ -201,8 +198,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class MinimumReduce extends GroupReduceFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class MinimumReduce extends RichGroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@@ -228,8 +224,7 @@ public class DependencyConnectedComponentsITCase extends JavaProgramTestBase {
}
}
- public static final class MinimumIdFilter extends FlatMapFunction
- <Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
+ public static final class MinimumIdFilter extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
index d719cc5..9ea33d7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/AggregatorsITCase.java
@@ -28,8 +28,8 @@ import org.junit.Assert;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
@@ -287,7 +287,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class SubtractOneMap extends MapFunction<Integer, Integer> {
+ public static final class SubtractOneMap extends RichMapFunction<Integer, Integer> {
private LongSumAggregator aggr;
@@ -309,7 +309,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class SubtractOneMapWithParam extends MapFunction<Integer, Integer> {
+ public static final class SubtractOneMapWithParam extends RichMapFunction<Integer, Integer> {
private LongSumAggregatorWithParameter aggr;
@@ -345,7 +345,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class TupleMakerMap extends MapFunction<Integer, Tuple2<Integer, Integer>> {
+ public static final class TupleMakerMap extends RichMapFunction<Integer, Tuple2<Integer, Integer>> {
@Override
public Tuple2<Integer, Integer> map(Integer value) throws Exception {
@@ -357,7 +357,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class AggregateMapDelta extends MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
+ public static final class AggregateMapDelta extends RichMapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
private LongSumAggregator aggr;
private LongValue previousAggr;
@@ -388,8 +388,8 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class UpdateFilter extends FlatMapFunction<Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>,
- Tuple2<Integer, Integer>> {
+ public static final class UpdateFilter extends RichFlatMapFunction<Tuple2<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>,
+ Tuple2<Integer, Integer>> {
private int superstep;
@@ -411,7 +411,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class ProjectSecondMapper extends MapFunction<Tuple2<Integer, Integer>, Integer> {
+ public static final class ProjectSecondMapper extends RichMapFunction<Tuple2<Integer, Integer>, Integer> {
@Override
public Integer map(Tuple2<Integer, Integer> value) {
@@ -420,7 +420,7 @@ public class AggregatorsITCase extends JavaProgramTestBase {
}
@SuppressWarnings("serial")
- public static final class AggregateMapDeltaWithParam extends MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
+ public static final class AggregateMapDeltaWithParam extends RichMapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>> {
private LongSumAggregatorWithParameter aggr;
private LongValue previousAggr;
[12/39] git commit: [FLINK-1039] Fix pojo expression keys for group
reduce
Posted by rm...@apache.org.
[FLINK-1039] Fix pojo expression keys for group reduce
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/23289d6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/23289d6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/23289d6e
Branch: refs/heads/travis_test
Commit: 23289d6eadeb7ad8798f41e747a4962745e35769
Parents: 32d168f
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Jul 31 02:32:37 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Jul 31 14:39:06 2014 +0200
----------------------------------------------------------------------
.../java/org/apache/flink/api/java/operators/Keys.java | 5 ++---
.../flink/api/java/operators/ReduceGroupOperator.java | 13 +++++++++++++
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/23289d6e/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
index 131fba9..2fdb520 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
@@ -197,8 +197,8 @@ public abstract class Keys<T> {
public ExpressionKeys(String[] expressions, TypeInformation<T> type) {
if (!(type instanceof PojoTypeInfo<?>)) {
throw new UnsupportedOperationException("Key expressions can only be used on POJOs." + " " +
- "A POCO must have a default constructor without arguments and not have readObject" +
- " and/or writeObject methods. Also, it can only have nested POJOs or primitive (also boxed)" +
+ "A POJO must have a default constructor without arguments and not have readObject" +
+ " and/or writeObject methods. A current restriction is that it can only have nested POJOs or primitive (also boxed)" +
" fields.");
}
PojoTypeInfo<?> pojoType = (PojoTypeInfo<?>) type;
@@ -212,7 +212,6 @@ public abstract class Keys<T> {
" type " + type.toString() + ".");
}
}
-
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/23289d6e/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
index 099860c..d88d43d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceGroupOperator.java
@@ -166,6 +166,19 @@ public class ReduceGroupOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return po;
}
+ else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
+
+ int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
+ UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
+ GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>> po =
+ new GroupReduceOperatorBase<IN, OUT, GenericGroupReduce<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
+
+ po.setCombinable(combinable);
+ po.setInput(input);
+ po.setDegreeOfParallelism(this.getParallelism());
+
+ return po;
+ }
else {
throw new UnsupportedOperationException("Unrecognized key type.");
}
[15/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
index 2f749d4..4c8177a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableAggregatorITCase.java
@@ -24,9 +24,9 @@ import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
@@ -147,8 +147,7 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
}
}
- public static final class NeighborWithComponentIDJoin extends JoinFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class NeighborWithComponentIDJoin extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
@@ -161,8 +160,7 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
}
}
- public static final class MinimumReduce extends GroupReduceFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class MinimumReduce extends RichGroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@@ -189,8 +187,7 @@ public class ConnectedComponentsWithParametrizableAggregatorITCase extends JavaP
}
@SuppressWarnings("serial")
- public static final class MinimumIdFilter extends FlatMapFunction
- <Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
+ public static final class MinimumIdFilter extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
private static LongSumAggregatorWithParameter aggr;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
index fa1676f..104c3df 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/aggregators/ConnectedComponentsWithParametrizableConvergenceITCase.java
@@ -25,9 +25,9 @@ import java.util.List;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.util.JavaProgramTestBase;
@@ -139,8 +139,7 @@ public class ConnectedComponentsWithParametrizableConvergenceITCase extends Java
}
}
- public static final class NeighborWithComponentIDJoin extends JoinFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class NeighborWithComponentIDJoin extends RichJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
@@ -153,8 +152,7 @@ public class ConnectedComponentsWithParametrizableConvergenceITCase extends Java
}
}
- public static final class MinimumReduce extends GroupReduceFunction
- <Tuple2<Long, Long>, Tuple2<Long, Long>> {
+ public static final class MinimumReduce extends RichGroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
private static final long serialVersionUID = 1L;
final Tuple2<Long, Long> resultVertex = new Tuple2<Long, Long>();
@@ -181,8 +179,7 @@ public class ConnectedComponentsWithParametrizableConvergenceITCase extends Java
}
@SuppressWarnings("serial")
- public static final class MinimumIdFilter extends FlatMapFunction
- <Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
+ public static final class MinimumIdFilter extends RichFlatMapFunction<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>, Tuple2<Long, Long>> {
private static LongSumAggregator aggr;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
index 0475a4f..1ec0eb4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductCoGroup.java
@@ -22,8 +22,8 @@ package org.apache.flink.test.iterative.nephele.customdanglingpagerank;
import java.util.Iterator;
import java.util.Set;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.iterative.nephele.ConfigUtils;
import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.VertexWithRank;
@@ -32,7 +32,7 @@ import org.apache.flink.test.iterative.nephele.danglingpagerank.PageRankStats;
import org.apache.flink.test.iterative.nephele.danglingpagerank.PageRankStatsAggregator;
import org.apache.flink.util.Collector;
-public class CustomCompensatableDotProductCoGroup extends AbstractFunction implements GenericCoGrouper<VertexWithRankAndDangling, VertexWithRank, VertexWithRankAndDangling> {
+public class CustomCompensatableDotProductCoGroup extends AbstractRichFunction implements CoGroupFunction<VertexWithRankAndDangling, VertexWithRank, VertexWithRankAndDangling> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductMatch.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductMatch.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductMatch.java
index 28c77ba..b44d914 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductMatch.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatableDotProductMatch.java
@@ -22,8 +22,8 @@ package org.apache.flink.test.iterative.nephele.customdanglingpagerank;
import java.util.Random;
import java.util.Set;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.iterative.nephele.ConfigUtils;
import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.VertexWithAdjacencyList;
@@ -31,8 +31,8 @@ import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.Vert
import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.VertexWithRankAndDangling;
import org.apache.flink.util.Collector;
-public class CustomCompensatableDotProductMatch extends AbstractFunction implements
- GenericJoiner<VertexWithRankAndDangling, VertexWithAdjacencyList, VertexWithRank>
+public class CustomCompensatableDotProductMatch extends AbstractRichFunction implements
+ FlatJoinFunction<VertexWithRankAndDangling, VertexWithAdjacencyList, VertexWithRank>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
index 74426c0..d83b33b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomCompensatingMap.java
@@ -21,7 +21,7 @@ package org.apache.flink.test.iterative.nephele.customdanglingpagerank;
import java.util.Set;
-import org.apache.flink.api.common.functions.AbstractFunction;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.GenericCollectorMap;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.iterative.nephele.ConfigUtils;
@@ -29,7 +29,7 @@ import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.Vert
import org.apache.flink.test.iterative.nephele.danglingpagerank.PageRankStats;
import org.apache.flink.util.Collector;
-public class CustomCompensatingMap extends AbstractFunction implements GenericCollectorMap<VertexWithRankAndDangling, VertexWithRankAndDangling> {
+public class CustomCompensatingMap extends AbstractRichFunction implements GenericCollectorMap<VertexWithRankAndDangling, VertexWithRankAndDangling> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
index 8af9247..1e08a9f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/CustomRankCombiner.java
@@ -20,15 +20,15 @@ package org.apache.flink.test.iterative.nephele.customdanglingpagerank;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.AbstractFunction;
-import org.apache.flink.api.common.functions.GenericCombine;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.FlatCombineFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.test.iterative.nephele.customdanglingpagerank.types.VertexWithRank;
import org.apache.flink.util.Collector;
-public class CustomRankCombiner extends AbstractFunction implements GenericGroupReduce<VertexWithRank, VertexWithRank>,
- GenericCombine<VertexWithRank>
+public class CustomRankCombiner extends AbstractRichFunction implements GroupReduceFunction<VertexWithRank, VertexWithRank>,
+ FlatCombineFunction<VertexWithRank>
{
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index b914c1c..3749c1d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -24,8 +24,9 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
@@ -301,7 +302,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
- public static class Tuple5CoGroup extends CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, Integer>> {
+ public static class Tuple5CoGroup implements CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, Integer>> {
private static final long serialVersionUID = 1L;
@@ -330,7 +331,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
}
- public static class CustomTypeCoGroup extends CoGroupFunction<CustomType, CustomType, CustomType> {
+ public static class CustomTypeCoGroup implements CoGroupFunction<CustomType, CustomType, CustomType> {
private static final long serialVersionUID = 1L;
@@ -358,7 +359,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
- public static class MixedCoGroup extends CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, CustomType, Tuple3<Integer, Long, String>> {
+ public static class MixedCoGroup implements CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, CustomType, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@@ -388,7 +389,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
- public static class MixedCoGroup2 extends CoGroupFunction<CustomType, Tuple5<Integer, Long, Integer, String, Long>, CustomType> {
+ public static class MixedCoGroup2 implements CoGroupFunction<CustomType, Tuple5<Integer, Long, Integer, String, Long>, CustomType> {
private static final long serialVersionUID = 1L;
@@ -417,7 +418,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
- public static class Tuple3ReturnLeft extends CoGroupFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
+ public static class Tuple3ReturnLeft implements CoGroupFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@@ -434,7 +435,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
}
- public static class Tuple5ReturnRight extends CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
+ public static class Tuple5ReturnRight implements CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
private static final long serialVersionUID = 1L;
@@ -456,7 +457,7 @@ public class CoGroupITCase extends JavaProgramTestBase {
}
- public static class Tuple5CoGroupBC extends CoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Integer, Integer>> {
+ public static class Tuple5CoGroupBC extends RichCoGroupFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Integer, Integer>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CrossITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CrossITCase.java
index dabe7fc..304dda2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CrossITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CrossITCase.java
@@ -23,7 +23,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.CrossFunction;
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.java.functions.RichCrossFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
@@ -400,7 +401,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
- public static class Tuple5Cross extends CrossFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, String>> {
+ public static class Tuple5Cross implements CrossFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, String>> {
private static final long serialVersionUID = 1L;
@@ -416,7 +417,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
- public static class CustomTypeCross extends CrossFunction<CustomType, CustomType, CustomType> {
+ public static class CustomTypeCross implements CrossFunction<CustomType, CustomType, CustomType> {
private static final long serialVersionUID = 1L;
@@ -429,7 +430,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
- public static class MixedCross extends CrossFunction<Tuple5<Integer, Long, Integer, String, Long>, CustomType, Tuple3<Integer, Long, String>> {
+ public static class MixedCross implements CrossFunction<Tuple5<Integer, Long, Integer, String, Long>, CustomType, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@@ -444,7 +445,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
- public static class Tuple3ReturnLeft extends CrossFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>> {
+ public static class Tuple3ReturnLeft implements CrossFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@@ -457,7 +458,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
}
- public static class Tuple5ReturnRight extends CrossFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
+ public static class Tuple5ReturnRight implements CrossFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
private static final long serialVersionUID = 1L;
@@ -473,7 +474,7 @@ public class CrossITCase extends JavaProgramTestBase {
}
- public static class Tuple5CrossBC extends CrossFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Integer, Integer>> {
+ public static class Tuple5CrossBC extends RichCrossFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Integer, Integer>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
index 203117c..0c6f3cc 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
@@ -23,7 +23,7 @@ import java.util.Collection;
import java.util.LinkedList;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
@@ -164,7 +164,7 @@ public class DistinctITCase extends JavaProgramTestBase {
return in.myInt;
}
})
- .map(new MapFunction<CollectionDataSets.CustomType, Tuple1<Integer>>() {
+ .map(new RichMapFunction<CustomType, Tuple1<Integer>>() {
@Override
public Tuple1<Integer> map(CustomType value) throws Exception {
return new Tuple1<Integer>(value.myInt);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
index 96174da..6613bc1 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FilterITCase.java
@@ -23,7 +23,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.FilterFunction;
+import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.java.functions.RichFilterFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
@@ -38,7 +39,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
@RunWith(Parameterized.class)
public class FilterITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 8;
+ private static int NUM_PROGRAMS = 8;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -268,7 +269,7 @@ public class FilterITCase extends JavaProgramTestBase {
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
- filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
+ filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
int literal = -1;
@@ -306,7 +307,7 @@ public class FilterITCase extends JavaProgramTestBase {
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
- filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
+ filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
private int broadcastSum = 0;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FlatMapITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FlatMapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FlatMapITCase.java
index 1c97347..a6dd377 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FlatMapITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/FlatMapITCase.java
@@ -23,7 +23,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
@@ -323,7 +324,7 @@ public class FlatMapITCase extends JavaProgramTestBase {
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> bcFlatMapDs = ds.
- flatMap(new FlatMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
+ flatMap(new RichFlatMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
private final Tuple3<Integer, Long, String> outTuple =
new Tuple3<Integer, Long, String>();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index 6556b5e..7376e86 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -24,10 +24,11 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
@@ -89,332 +90,336 @@ public class GroupReduceITCase extends JavaProgramTestBase {
private static class GroupReduceProgs {
public static String runProgram(int progId, String resultPath) throws Exception {
-
- switch(progId) {
- case 1: {
+
+ switch (progId) {
+ case 1: {
/*
* check correctness of groupReduce on tuples with key field selector
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple2<Integer, Long>> reduceDs = ds.
- groupBy(1).reduceGroup(new Tuple3GroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1\n" +
- "5,2\n" +
- "15,3\n" +
- "34,4\n" +
- "65,5\n" +
- "111,6\n";
- }
- case 2: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple2<Integer, Long>> reduceDs = ds.
+ groupBy(1).reduceGroup(new Tuple3GroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1\n" +
+ "5,2\n" +
+ "15,3\n" +
+ "34,4\n" +
+ "65,5\n" +
+ "111,6\n";
+ }
+ case 2: {
/*
* check correctness of groupReduce on tuples with multiple key field selector
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
- DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds.
- groupBy(4,0).reduceGroup(new Tuple5GroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1,0,P-),1\n" +
- "2,3,0,P-),1\n" +
- "2,2,0,P-),2\n" +
- "3,9,0,P-),2\n" +
- "3,6,0,P-),3\n" +
- "4,17,0,P-),1\n" +
- "4,17,0,P-),2\n" +
- "5,11,0,P-),1\n" +
- "5,29,0,P-),2\n" +
- "5,25,0,P-),3\n";
- }
- case 3: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds.
+ groupBy(4, 0).reduceGroup(new Tuple5GroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,0,P-),1\n" +
+ "2,3,0,P-),1\n" +
+ "2,2,0,P-),2\n" +
+ "3,9,0,P-),2\n" +
+ "3,6,0,P-),3\n" +
+ "4,17,0,P-),1\n" +
+ "4,17,0,P-),2\n" +
+ "5,11,0,P-),1\n" +
+ "5,29,0,P-),2\n" +
+ "5,25,0,P-),3\n";
+ }
+ case 3: {
/*
* check correctness of groupReduce on tuples with key field selector and group sorting
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(1);
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
- groupBy(1).sortGroup(2,Order.ASCENDING).reduceGroup(new Tuple3SortedGroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1,Hi\n" +
- "5,2,Hello-Hello world\n" +
- "15,3,Hello world, how are you?-I am fine.-Luke Skywalker\n" +
- "34,4,Comment#1-Comment#2-Comment#3-Comment#4\n" +
- "65,5,Comment#5-Comment#6-Comment#7-Comment#8-Comment#9\n" +
- "111,6,Comment#10-Comment#11-Comment#12-Comment#13-Comment#14-Comment#15\n";
-
- }
- case 4: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
+ groupBy(1).sortGroup(2, Order.ASCENDING).reduceGroup(new Tuple3SortedGroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,Hi\n" +
+ "5,2,Hello-Hello world\n" +
+ "15,3,Hello world, how are you?-I am fine.-Luke Skywalker\n" +
+ "34,4,Comment#1-Comment#2-Comment#3-Comment#4\n" +
+ "65,5,Comment#5-Comment#6-Comment#7-Comment#8-Comment#9\n" +
+ "111,6,Comment#10-Comment#11-Comment#12-Comment#13-Comment#14-Comment#15\n";
+
+ }
+ case 4: {
/*
* check correctness of groupReduce on tuples with key extractor
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple2<Integer, Long>> reduceDs = ds.
- groupBy(new KeySelector<Tuple3<Integer,Long,String>, Long>() {
- private static final long serialVersionUID = 1L;
- @Override
- public Long getKey(Tuple3<Integer, Long, String> in) {
- return in.f1;
- }
- }).reduceGroup(new Tuple3GroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1\n" +
- "5,2\n" +
- "15,3\n" +
- "34,4\n" +
- "65,5\n" +
- "111,6\n";
-
- }
- case 5: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple2<Integer, Long>> reduceDs = ds.
+ groupBy(new KeySelector<Tuple3<Integer, Long, String>, Long>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Long getKey(Tuple3<Integer, Long, String> in) {
+ return in.f1;
+ }
+ }).reduceGroup(new Tuple3GroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1\n" +
+ "5,2\n" +
+ "15,3\n" +
+ "34,4\n" +
+ "65,5\n" +
+ "111,6\n";
+
+ }
+ case 5: {
/*
* check correctness of groupReduce on custom type with type extractor
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
- DataSet<CustomType> reduceDs = ds.
- groupBy(new KeySelector<CustomType, Integer>() {
- private static final long serialVersionUID = 1L;
- @Override
- public Integer getKey(CustomType in) {
- return in.myInt;
- }
- }).reduceGroup(new CustomTypeGroupReduce());
-
- reduceDs.writeAsText(resultPath);
- env.execute();
-
- // return expected result
- return "1,0,Hello!\n" +
- "2,3,Hello!\n" +
- "3,12,Hello!\n" +
- "4,30,Hello!\n" +
- "5,60,Hello!\n" +
- "6,105,Hello!\n";
- }
- case 6: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
+ DataSet<CustomType> reduceDs = ds.
+ groupBy(new KeySelector<CustomType, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(CustomType in) {
+ return in.myInt;
+ }
+ }).reduceGroup(new CustomTypeGroupReduce());
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,0,Hello!\n" +
+ "2,3,Hello!\n" +
+ "3,12,Hello!\n" +
+ "4,30,Hello!\n" +
+ "5,60,Hello!\n" +
+ "6,105,Hello!\n";
+ }
+ case 6: {
/*
* check correctness of all-groupreduce for tuples
*/
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.reduceGroup(new AllAddingTuple3GroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "231,91,Hello World\n";
- }
- case 7: {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.reduceGroup(new AllAddingTuple3GroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "231,91,Hello World\n";
+ }
+ case 7: {
/*
* check correctness of all-groupreduce for custom types
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
- DataSet<CustomType> reduceDs = ds.reduceGroup(new AllAddingCustomTypeGroupReduce());
-
- reduceDs.writeAsText(resultPath);
- env.execute();
-
- // return expected result
- return "91,210,Hello!";
- }
- case 8: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
+ DataSet<CustomType> reduceDs = ds.reduceGroup(new AllAddingCustomTypeGroupReduce());
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "91,210,Hello!";
+ }
+ case 8: {
/*
* check correctness of groupReduce with broadcast set
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
- groupBy(1).reduceGroup(new BCTuple3GroupReduce()).withBroadcastSet(intDs, "ints");
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1,55\n" +
- "5,2,55\n" +
- "15,3,55\n" +
- "34,4,55\n" +
- "65,5,55\n" +
- "111,6,55\n";
- }
- case 9: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
+ groupBy(1).reduceGroup(new BCTuple3GroupReduce()).withBroadcastSet(intDs, "ints");
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,55\n" +
+ "5,2,55\n" +
+ "15,3,55\n" +
+ "34,4,55\n" +
+ "65,5,55\n" +
+ "111,6,55\n";
+ }
+ case 9: {
/*
* check correctness of groupReduce if UDF returns input objects multiple times and changes it in between
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
- groupBy(1).reduceGroup(new InputReturningTuple3GroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "11,1,Hi!\n" +
- "21,1,Hi again!\n" +
- "12,2,Hi!\n" +
- "22,2,Hi again!\n" +
- "13,2,Hi!\n" +
- "23,2,Hi again!\n";
- }
- case 10: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
+ groupBy(1).reduceGroup(new InputReturningTuple3GroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "11,1,Hi!\n" +
+ "21,1,Hi again!\n" +
+ "12,2,Hi!\n" +
+ "22,2,Hi again!\n" +
+ "13,2,Hi!\n" +
+ "23,2,Hi again!\n";
+ }
+ case 10: {
/*
* check correctness of groupReduce on custom type with key extractor and combine
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
- DataSet<CustomType> reduceDs = ds.
- groupBy(new KeySelector<CustomType, Integer>() {
- private static final long serialVersionUID = 1L;
- @Override
- public Integer getKey(CustomType in) {
- return in.myInt;
- }
- }).reduceGroup(new CustomTypeGroupReduceWithCombine());
-
- reduceDs.writeAsText(resultPath);
- env.execute();
-
- // return expected result
- return "1,0,test1\n" +
- "2,3,test2\n" +
- "3,12,test3\n" +
- "4,30,test4\n" +
- "5,60,test5\n" +
- "6,105,test6\n";
- }
- case 11: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
+ DataSet<CustomType> reduceDs = ds.
+ groupBy(new KeySelector<CustomType, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(CustomType in) {
+ return in.myInt;
+ }
+ }).reduceGroup(new CustomTypeGroupReduceWithCombine());
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,0,test1\n" +
+ "2,3,test2\n" +
+ "3,12,test3\n" +
+ "4,30,test4\n" +
+ "5,60,test5\n" +
+ "6,105,test6\n";
+ }
+ case 11: {
/*
* check correctness of groupReduce on tuples with combine
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(2); // important because it determines how often the combiner is called
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple2<Integer, String>> reduceDs = ds.
- groupBy(1).reduceGroup(new Tuple3GroupReduceWithCombine());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,test1\n" +
- "5,test2\n" +
- "15,test3\n" +
- "34,test4\n" +
- "65,test5\n" +
- "111,test6\n";
- }
- // all-groupreduce with combine
- case 12: {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(2); // important because it determines how often the combiner is called
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple2<Integer, String>> reduceDs = ds.
+ groupBy(1).reduceGroup(new Tuple3GroupReduceWithCombine());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,test1\n" +
+ "5,test2\n" +
+ "15,test3\n" +
+ "34,test4\n" +
+ "65,test5\n" +
+ "111,test6\n";
+ }
+ // all-groupreduce with combine
+ case 12: {
/*
* check correctness of all-groupreduce for tuples with combine
*/
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env)
- .map(new IdentityMapper<Tuple3<Integer,Long,String>>()).setParallelism(4);
-
- Configuration cfg = new Configuration();
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION);
- DataSet<Tuple2<Integer, String>> reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine())
- .withParameters(cfg);
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "322,testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest\n";
- }
- // descending sort not working
- case 13: {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env)
+ .map(new IdentityMapper<Tuple3<Integer, Long, String>>()).setParallelism(4);
+
+ Configuration cfg = new Configuration();
+ cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION);
+ DataSet<Tuple2<Integer, String>> reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine())
+ .withParameters(cfg);
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "322,testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest\n";
+ }
+ // descending sort not working
+ case 13: {
/*
* check correctness of groupReduce on tuples with key field selector and group sorting
*/
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- env.setDegreeOfParallelism(1);
-
- DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
- groupBy(1).sortGroup(2,Order.DESCENDING).reduceGroup(new Tuple3SortedGroupReduce());
-
- reduceDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "1,1,Hi\n" +
- "5,2,Hello world-Hello\n" +
- "15,3,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
- "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" +
- "65,5,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
- "111,6,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";
-
- }
- default:
- throw new IllegalArgumentException("Invalid program id");
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
+ groupBy(1).sortGroup(2, Order.DESCENDING).reduceGroup(new Tuple3SortedGroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,Hi\n" +
+ "5,2,Hello world-Hello\n" +
+ "15,3,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
+ "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" +
+ "65,5,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
+ "111,6,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";
+
+ }
+ default: {
+ throw new IllegalArgumentException("Invalid program id");
+ }
}
}
}
- public static class Tuple3GroupReduce extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>> {
+ public static class Tuple3GroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>> {
private static final long serialVersionUID = 1L;
@@ -436,7 +441,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class Tuple3SortedGroupReduce extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
+ public static class Tuple3SortedGroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@@ -462,7 +467,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class Tuple5GroupReduce extends GroupReduceFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
+ public static class Tuple5GroupReduce implements GroupReduceFunction<Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
private static final long serialVersionUID = 1L;
@Override
@@ -486,7 +491,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class CustomTypeGroupReduce extends GroupReduceFunction<CustomType, CustomType> {
+ public static class CustomTypeGroupReduce implements GroupReduceFunction<CustomType, CustomType> {
private static final long serialVersionUID = 1L;
@@ -511,8 +516,9 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
-
- public static class InputReturningTuple3GroupReduce extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
+
+
+ public static class InputReturningTuple3GroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@Override
@@ -534,7 +540,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class AllAddingTuple3GroupReduce extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
+ public static class AllAddingTuple3GroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@Override
@@ -554,7 +560,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class AllAddingCustomTypeGroupReduce extends GroupReduceFunction<CustomType, CustomType> {
+ public static class AllAddingCustomTypeGroupReduce implements GroupReduceFunction<CustomType, CustomType> {
private static final long serialVersionUID = 1L;
@Override
@@ -579,7 +585,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static class BCTuple3GroupReduce extends GroupReduceFunction<Tuple3<Integer, Long, String>,Tuple3<Integer, Long, String>> {
+ public static class BCTuple3GroupReduce extends RichGroupReduceFunction<Tuple3<Integer, Long, String>,Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
private String f2Replace = "";
@@ -613,8 +619,8 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- @org.apache.flink.api.java.functions.GroupReduceFunction.Combinable
- public static class Tuple3GroupReduceWithCombine extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, String>> {
+ @RichGroupReduceFunction.Combinable
+ public static class Tuple3GroupReduceWithCombine extends RichGroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, String>> {
private static final long serialVersionUID = 1L;
@Override
@@ -650,8 +656,8 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- @org.apache.flink.api.java.functions.GroupReduceFunction.Combinable
- public static class Tuple3AllGroupReduceWithCombine extends GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, String>> {
+ @RichGroupReduceFunction.Combinable
+ public static class Tuple3AllGroupReduceWithCombine extends RichGroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, String>> {
private static final long serialVersionUID = 1L;
@Override
@@ -686,8 +692,8 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- @org.apache.flink.api.java.functions.GroupReduceFunction.Combinable
- public static class CustomTypeGroupReduceWithCombine extends GroupReduceFunction<CustomType, CustomType> {
+ @RichGroupReduceFunction.Combinable
+ public static class CustomTypeGroupReduceWithCombine extends RichGroupReduceFunction<CustomType, CustomType> {
private static final long serialVersionUID = 1L;
@Override
@@ -723,7 +729,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
}
- public static final class IdentityMapper<T> extends MapFunction<T, T> {
+ public static final class IdentityMapper<T> extends RichMapFunction<T, T> {
@Override
public T map(T value) { return value; }
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
index ffad949..a293cbf 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
@@ -23,8 +23,10 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
@@ -33,6 +35,7 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
import org.apache.flink.test.util.JavaProgramTestBase;
+import org.apache.flink.util.Collector;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -101,7 +104,7 @@ public class JoinITCase extends JavaProgramTestBase {
ds1.join(ds2)
.where(1)
.equalTo(1)
- .with(new T3T5Join());
+ .with(new T3T5FlatJoin());
joinDs.writeAsCsv(resultPath);
env.execute();
@@ -126,7 +129,7 @@ public class JoinITCase extends JavaProgramTestBase {
ds1.join(ds2)
.where(0,1)
.equalTo(0,4)
- .with(new T3T5Join());
+ .with(new T3T5FlatJoin());
joinDs.writeAsCsv(resultPath);
env.execute();
@@ -177,7 +180,7 @@ public class JoinITCase extends JavaProgramTestBase {
DataSet<Tuple2<String, String>> joinDs = ds1.joinWithHuge(ds2)
.where(1)
.equalTo(1)
- .with(new T3T5Join());
+ .with(new T3T5FlatJoin());
joinDs.writeAsCsv(resultPath);
env.execute();
@@ -202,7 +205,7 @@ public class JoinITCase extends JavaProgramTestBase {
ds1.joinWithTiny(ds2)
.where(1)
.equalTo(1)
- .with(new T3T5Join());
+ .with(new T3T5FlatJoin());
joinDs.writeAsCsv(resultPath);
env.execute();
@@ -292,35 +295,35 @@ public class JoinITCase extends JavaProgramTestBase {
}
case 9: {
- /*
- * Join on a tuple input with key field selector and a custom type input with key extractor
- */
-
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ /*
+ * Join on a tuple input with key field selector and a custom type input with key extractor
+ */
- DataSet<CustomType> ds1 = CollectionDataSets.getSmallCustomTypeDataSet(env);
- DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
- DataSet<Tuple2<String, String>> joinDs =
- ds1.join(ds2)
- .where(new KeySelector<CustomType, Integer>() {
- @Override
- public Integer getKey(CustomType value) {
- return value.myInt;
- }
- }
- )
- .equalTo(0)
- .with(new CustT3Join());
-
- joinDs.writeAsCsv(resultPath);
- env.execute();
-
- // return expected result
- return "Hi,Hi\n" +
- "Hello,Hello\n" +
- "Hello world,Hello\n";
-
- }
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CustomType> ds1 = CollectionDataSets.getSmallCustomTypeDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple2<String, String>> joinDs =
+ ds1.join(ds2)
+ .where(new KeySelector<CustomType, Integer>() {
+ @Override
+ public Integer getKey(CustomType value) {
+ return value.myInt;
+ }
+ }
+ )
+ .equalTo(0)
+ .with(new CustT3Join());
+
+ joinDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "Hi,Hi\n" +
+ "Hello,Hello\n" +
+ "Hello world,Hello\n";
+
+ }
case 10: {
/*
@@ -458,38 +461,39 @@ public class JoinITCase extends JavaProgramTestBase {
}
- public static class T3T5Join extends JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<String, String>> {
+ public static class T3T5FlatJoin implements FlatJoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<String, String>> {
@Override
- public Tuple2<String, String> join(Tuple3<Integer, Long, String> first,
- Tuple5<Integer, Long, Integer, String, Long> second) {
-
- return new Tuple2<String,String>(first.f2, second.f3);
+ public void join(Tuple3<Integer, Long, String> first,
+ Tuple5<Integer, Long, Integer, String, Long> second,
+ Collector<Tuple2<String,String>> out) {
+
+ out.collect (new Tuple2<String,String> (first.f2, second.f3));
}
-
+
}
- public static class LeftReturningJoin extends JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>> {
+ public static class LeftReturningJoin implements JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<Integer, Long, String>> {
@Override
public Tuple3<Integer, Long, String> join(Tuple3<Integer, Long, String> first,
- Tuple5<Integer, Long, Integer, String, Long> second) {
+ Tuple5<Integer, Long, Integer, String, Long> second) {
return first;
}
}
- public static class RightReturningJoin extends JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
+ public static class RightReturningJoin implements JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple5<Integer, Long, Integer, String, Long>> {
@Override
public Tuple5<Integer, Long, Integer, String, Long> join(Tuple3<Integer, Long, String> first,
- Tuple5<Integer, Long, Integer, String, Long> second) {
+ Tuple5<Integer, Long, Integer, String, Long> second) {
return second;
}
}
- public static class T3T5BCJoin extends JoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<String, String, Integer>> {
+ public static class T3T5BCJoin extends RichFlatJoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple3<String, String, Integer>> {
private int broadcast;
@@ -505,6 +509,7 @@ public class JoinITCase extends JavaProgramTestBase {
}
+ /*
@Override
public Tuple3<String, String, Integer> join(
Tuple3<Integer, Long, String> first,
@@ -512,19 +517,25 @@ public class JoinITCase extends JavaProgramTestBase {
return new Tuple3<String, String, Integer>(first.f2, second.f3, broadcast);
}
+ */
+
+ @Override
+ public void join(Tuple3<Integer, Long, String> first, Tuple5<Integer, Long, Integer, String, Long> second, Collector<Tuple3<String, String, Integer>> out) throws Exception {
+ out.collect(new Tuple3<String, String, Integer> (first.f2, second.f3, broadcast));
+ }
}
- public static class T3CustJoin extends JoinFunction<Tuple3<Integer, Long, String>, CustomType, Tuple2<String, String>> {
+ public static class T3CustJoin implements JoinFunction<Tuple3<Integer, Long, String>, CustomType, Tuple2<String, String>> {
@Override
public Tuple2<String, String> join(Tuple3<Integer, Long, String> first,
- CustomType second) {
+ CustomType second) {
return new Tuple2<String, String>(first.f2, second.myString);
}
}
- public static class CustT3Join extends JoinFunction<CustomType, Tuple3<Integer, Long, String>, Tuple2<String, String>> {
+ public static class CustT3Join implements JoinFunction<CustomType, Tuple3<Integer, Long, String>, Tuple2<String, String>> {
@Override
public Tuple2<String, String> join(CustomType first, Tuple3<Integer, Long, String> second) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
index 0921e82..4f1fb1a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/MapITCase.java
@@ -25,7 +25,8 @@ import java.util.LinkedList;
import org.junit.Assert;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
@@ -396,7 +397,7 @@ public class MapITCase extends JavaProgramTestBase {
DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.
- map(new MapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
+ map(new RichMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
private Integer f2Replace = 0;
@@ -457,7 +458,7 @@ public class MapITCase extends JavaProgramTestBase {
final int testValue = 666;
conf.setInteger(testKey, testValue);
DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.
- map(new MapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
+ map(new RichMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
index 6cc1061..a296a09 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
@@ -23,8 +23,9 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
+import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.configuration.Configuration;
@@ -270,7 +271,7 @@ public class ReduceITCase extends JavaProgramTestBase {
"65,5,Hi again!\n" +
"111,6,Hi again!\n";
}
- default:
+ default:
throw new IllegalArgumentException("Invalid program id");
}
@@ -278,7 +279,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
- public static class Tuple3Reduce extends ReduceFunction<Tuple3<Integer, Long, String>> {
+ public static class Tuple3Reduce implements ReduceFunction<Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
private final String f2Replace;
@@ -306,7 +307,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class Tuple5Reduce extends ReduceFunction<Tuple5<Integer, Long, Integer, String, Long>> {
+ public static class Tuple5Reduce implements ReduceFunction<Tuple5<Integer, Long, Integer, String, Long>> {
private static final long serialVersionUID = 1L;
private final Tuple5<Integer, Long, Integer, String, Long> out = new Tuple5<Integer, Long, Integer, String, Long>();
@@ -321,7 +322,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class CustomTypeReduce extends ReduceFunction<CustomType> {
+ public static class CustomTypeReduce implements ReduceFunction<CustomType> {
private static final long serialVersionUID = 1L;
private final CustomType out = new CustomType();
@@ -336,7 +337,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class InputReturningTuple3Reduce extends ReduceFunction<Tuple3<Integer, Long, String>> {
+ public static class InputReturningTuple3Reduce implements ReduceFunction<Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
@Override
@@ -350,7 +351,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class AllAddingTuple3Reduce extends ReduceFunction<Tuple3<Integer, Long, String>> {
+ public static class AllAddingTuple3Reduce implements ReduceFunction<Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
@@ -364,7 +365,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class AllAddingCustomTypeReduce extends ReduceFunction<CustomType> {
+ public static class AllAddingCustomTypeReduce implements ReduceFunction<CustomType> {
private static final long serialVersionUID = 1L;
private final CustomType out = new CustomType();
@@ -379,7 +380,7 @@ public class ReduceITCase extends JavaProgramTestBase {
}
}
- public static class BCTuple3Reduce extends ReduceFunction<Tuple3<Integer, Long, String>> {
+ public static class BCTuple3Reduce extends RichReduceFunction<Tuple3<Integer, Long, String>> {
private static final long serialVersionUID = 1L;
private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
private String f2Replace = "";
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/UnionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/UnionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/UnionITCase.java
index 09191cc..a636ba4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/UnionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/UnionITCase.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
-import org.apache.flink.api.java.functions.FilterFunction;
+import org.apache.flink.api.java.functions.RichFilterFunction;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
@@ -145,7 +145,7 @@ public class UnionITCase extends JavaProgramTestBase {
// Don't know how to make an empty result in an other way than filtering it
DataSet<Tuple3<Integer, Long, String>> empty = CollectionDataSets.get3TupleDataSet(env).
- filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
+ filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
index ed573be..aaad08c 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/operators/CrossITCase.java
@@ -36,6 +36,7 @@ import org.apache.flink.types.IntValue;
import org.apache.flink.types.Record;
import org.apache.flink.types.StringValue;
import org.apache.flink.util.Collector;
+import org.junit.Ignore;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -49,6 +50,7 @@ import java.util.LinkedList;
/**
*/
@RunWith(Parameterized.class)
+//@Ignore("Test needs to be adapted to new cross signature")
public class CrossITCase extends RecordAPITestBase {
private static final Log LOG = LogFactory.getLog(CrossITCase.class);
@@ -61,13 +63,30 @@ public class CrossITCase extends RecordAPITestBase {
super(testConfig);
}
- private static final String LEFT_IN = "1 1\n2 2\n1 1\n2 2\n3 3\n4 4\n3 3\n4 4\n";
+ //private static final String LEFT_IN = "1 1\n2 2\n1 1\n2 2\n3 3\n4 4\n3 3\n4 4\n";
- private static final String RIGHT_IN = "1 1\n1 2\n2 2\n2 4\n3 3\n3 6\n4 4\n4 8\n";
+ //private static final String RIGHT_IN = "1 1\n1 2\n2 2\n2 4\n3 3\n3 6\n4 4\n4 8\n";
- private static final String RESULT = "4 1\n4 1\n4 2\n4 2\n5 2\n5 2\n5 4\n5 4\n6 3\n6 3\n7 4\n7 4\n"
- + "5 0\n5 0\n5 1\n5 1\n6 1\n6 1\n6 3\n6 3\n7 2\n7 2\n8 3\n8 3\n"
- + "6 -1\n6 -1\n6 0\n6 0\n7 0\n7 0\n8 1\n8 1\n" + "7 -2\n7 -2\n7 -1\n7 -1\n8 -1\n8 -1\n";
+ //private static final String RESULT = "4 1\n4 1\n4 2\n4 2\n5 2\n5 2\n5 4\n5 4\n6 3\n6 3\n7 4\n7 4\n"
+ // + "5 0\n5 0\n5 1\n5 1\n6 1\n6 1\n6 3\n6 3\n7 2\n7 2\n8 3\n8 3\n"
+ // + "6 -1\n6 -1\n6 0\n6 0\n7 0\n7 0\n8 1\n8 1\n" + "7 -2\n7 -2\n7 -1\n7 -1\n8 -1\n8 -1\n";
+
+ //private static final String RESULT = "10 1\n10 1\n10 5\n10 5\n4 1\n4 1\n4 2\n4 2\n5 0\n5 0\n5 1\n," +
+ // "5 1\n5 2\n5 2\n5 4\n5 4\n6 -1\n6 -1\n6 0\n6 0\n6 1\n6 1\n6 3\n6 3\n6 3\n6 3\n6 6\n6 6\n7 -1\n" +
+ // "7 -1\n7 -2\n7 -2\n7 0\n7 0\n7 2\n7 2\n7 2\n7 2\n7 4\n7 4\n7 5\n7 5\n7 8\n7 8\n8 -1\n8 -1\n8 1\n" +
+ // "8 1\n8 1\n8 1\n8 3\n8 3\n8 4\n8 4\n8 7\n8 7\n9 0\n9 0\n9 2\n9 2\n9 3\n9 3\n9 6\n9 6\n";
+
+ //private static final String RESULT = "2 2\n4 4\n1 1\n3 3\n2 2\n4 4\n1 1\n3 3\n5 0\n5 1\n6 1\n 6 3\n" +
+ // "7 2\n7 5\n8 3\n8 7\n7 -2\n7 -1\n8 -1\n8 1\n9 0\n9 3\n10 1\n10 5\n4 1\n4 2\n5 2\n5 4\n6 3\n" +
+ // "6 6\n7 4\n7 8\n6 -1\n6 0\n7 0\n7 2\n8 1\n8 4\n9 2\n9 6\n5 0\n5 1\n6 1\n6 3\n7 2\n7 5\n 8 3\n" +
+ // "8 7\n7 -2\n7 -1\n8 -1\n8 1\n9 0\n9 3\n10 1\n10 5\n4 1\n4 2\n5 2\n5 4\n6 3\n6 6\n7 4\n7 8\n" +
+ // "6 -1\n6 0\n7 0\n7 2\n8 1\n8 4\n9 2\n9 6";
+
+
+ private static final String LEFT_IN = "1 1\n2 2\n3 3\n";
+ private static final String RIGHT_IN = "3 6\n4 4\n4 8\n";
+
+ private static final String RESULT = "6 6\n7 5\n7 8\n7 4\n8 3\n8 7\n8 4\n9 2\n9 6\n";
@Override
protected void preSubmit() throws Exception {
@@ -84,7 +103,7 @@ public class CrossITCase extends RecordAPITestBase {
private IntValue integer = new IntValue();
@Override
- public void cross(Record record1, Record record2, Collector<Record> out) {
+ public Record cross(Record record1, Record record2) throws Exception {
string = record1.getField(1, string);
int val1 = Integer.parseInt(string.toString());
string = record2.getField(1, string);
@@ -95,16 +114,14 @@ public class CrossITCase extends RecordAPITestBase {
int key2 = Integer.parseInt(string.toString());
LOG.debug("Processing { [" + key1 + "," + val1 + "] , [" + key2 + "," + val2 + "] }");
-
- if (val1 + val2 <= 6) {
- string.setValue((key1 + key2 + 2) + "");
- integer.setValue(val2 - val1 + 1);
-
- record1.setField(0, string);
- record1.setField(1, integer);
-
- out.collect(record1);
- }
+
+ string.setValue((key1 + key2 + 2) + "");
+ integer.setValue(val2 - val1 + 1);
+
+ record1.setField(0, string);
+ record1.setField(1, integer);
+
+ return record1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
index a545e05..0f58d18 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
@@ -47,19 +47,19 @@ public class ComputeDistance extends CrossFunction implements Serializable {
* 3: distance
*/
@Override
- public void cross(Record dataPointRecord, Record clusterCenterRecord, Collector<Record> out) {
-
+ public Record cross(Record dataPointRecord, Record clusterCenterRecord) throws Exception {
+
CoordVector dataPoint = dataPointRecord.getField(1, CoordVector.class);
-
+
IntValue clusterCenterId = clusterCenterRecord.getField(0, IntValue.class);
CoordVector clusterPoint = clusterCenterRecord.getField(1, CoordVector.class);
-
+
this.distance.setValue(dataPoint.computeEuclidianDistance(clusterPoint));
-
- // add cluster center id and distance to the data point record
+
+ // add cluster center id and distance to the data point record
dataPointRecord.setField(2, clusterCenterId);
dataPointRecord.setField(3, this.distance);
-
- out.collect(dataPointRecord);
+
+ return dataPointRecord;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/recordJobs/relational/TPCHQuery3.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/relational/TPCHQuery3.java b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/relational/TPCHQuery3.java
index 15640c0..441dc39 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/relational/TPCHQuery3.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/relational/TPCHQuery3.java
@@ -89,7 +89,7 @@ public class TPCHQuery3 implements Program, ProgramDescription {
/**
* Reads the filter literals from the configuration.
*
- * @see org.apache.flink.api.common.functions.Function#open(org.apache.flink.configuration.Configuration)
+ * @see org.apache.flink.api.common.functions.RichFunction#open(org.apache.flink.configuration.Configuration)
*/
@Override
public void open(Configuration parameters) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
index 97367f7..7149cd3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
@@ -24,8 +24,8 @@ import java.util.Collection;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.Program;
-import org.apache.flink.api.java.functions.MapFunction;
-import org.apache.flink.api.java.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
@@ -170,7 +170,7 @@ public class KMeansForTest implements Program {
// *************************************************************************
/** Converts a Tuple2<Double,Double> into a Point. */
- public static final class TuplePointConverter extends MapFunction<Tuple2<Double, Double>, Point> {
+ public static final class TuplePointConverter extends RichMapFunction<Tuple2<Double, Double>, Point> {
@Override
public Point map(Tuple2<Double, Double> t) throws Exception {
@@ -179,7 +179,7 @@ public class KMeansForTest implements Program {
}
/** Converts a Tuple3<Integer, Double,Double> into a Centroid. */
- public static final class TupleCentroidConverter extends MapFunction<Tuple3<Integer, Double, Double>, Centroid> {
+ public static final class TupleCentroidConverter extends RichMapFunction<Tuple3<Integer, Double, Double>, Centroid> {
@Override
public Centroid map(Tuple3<Integer, Double, Double> t) throws Exception {
@@ -188,7 +188,7 @@ public class KMeansForTest implements Program {
}
/** Determines the closest cluster center for a data point. */
- public static final class SelectNearestCenter extends MapFunction<Point, Tuple2<Integer, Point>> {
+ public static final class SelectNearestCenter extends RichMapFunction<Point, Tuple2<Integer, Point>> {
private Collection<Centroid> centroids;
/** Reads the centroid values from a broadcast variable into a collection. */
@@ -236,7 +236,7 @@ public class KMeansForTest implements Program {
}
/** Appends a count variable to the tuple. */
- public static final class CountAppender extends MapFunction<Tuple2<Integer, Point>, DummyTuple3IntPointLong> {
+ public static final class CountAppender extends RichMapFunction<Tuple2<Integer, Point>, DummyTuple3IntPointLong> {
@Override
public DummyTuple3IntPointLong map(Tuple2<Integer, Point> t) {
@@ -245,7 +245,7 @@ public class KMeansForTest implements Program {
}
/** Sums and counts point coordinates. */
- public static final class CentroidAccumulator extends ReduceFunction<DummyTuple3IntPointLong> {
+ public static final class CentroidAccumulator extends RichReduceFunction<DummyTuple3IntPointLong> {
@Override
public DummyTuple3IntPointLong reduce(DummyTuple3IntPointLong val1, DummyTuple3IntPointLong val2) {
@@ -254,7 +254,7 @@ public class KMeansForTest implements Program {
}
/** Computes new centroid from coordinate sum and count of points. */
- public static final class CentroidAverager extends MapFunction<DummyTuple3IntPointLong, Centroid> {
+ public static final class CentroidAverager extends RichMapFunction<DummyTuple3IntPointLong, Centroid> {
@Override
public Centroid map(DummyTuple3IntPointLong value) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7e0f153..8058539 100644
--- a/pom.xml
+++ b/pom.xml
@@ -274,6 +274,9 @@ under the License.
<activeByDefault>false</activeByDefault>
<jdk>1.8</jdk>
</activation>
+ <modules>
+ <module>flink-java8-tests</module>
+ </modules>
<build>
<plugins>
<plugin>
[11/39] git commit: Remove unused import from
OutboundEnvelopeEncoderTest
Posted by rm...@apache.org.
Remove unused import from OutboundEnvelopeEncoderTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/32d168f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/32d168f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/32d168f4
Branch: refs/heads/travis_test
Commit: 32d168f439bdb5dfab02a3ab2d12e87d0622a67e
Parents: ccd4950
Author: uce <u....@fu-berlin.de>
Authored: Thu Jul 31 14:16:10 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Thu Jul 31 14:16:10 2014 +0200
----------------------------------------------------------------------
.../flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/32d168f4/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
index 7ddf4c8..ef7c9b9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/OutboundEnvelopeEncoderTest.java
@@ -28,7 +28,6 @@ import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.network.Buffer;
import org.apache.flink.runtime.io.network.Envelope;
import org.apache.flink.runtime.io.network.channels.ChannelID;
-import org.apache.flink.runtime.io.network.netty.OutboundEnvelopeEncoder;
import org.apache.flink.runtime.jobgraph.JobID;
import org.junit.Test;
[05/39] git commit: [Documentation] Move "How to contrib";
"coding guidelines" out of the code repo (part of FLINK-1012) Improve
build instructions with respect to Hadoop 1 and Hadoop 2 dependencies. [ci
skip]
Posted by rm...@apache.org.
[Documentation] Move "How to contrib"; "coding guidelines" out of the code repo (part of FLINK-1012)
Improve build instructions with respect to Hadoop 1 and Hadoop 2 dependencies.
[ci skip]
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/25b4099d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/25b4099d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/25b4099d
Branch: refs/heads/travis_test
Commit: 25b4099d3af85ae881aaa52ddae0d71f37e67b4c
Parents: 2e50eac
Author: Robert Metzger <rm...@apache.org>
Authored: Sat Jul 26 19:05:49 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Sat Jul 26 19:06:44 2014 +0200
----------------------------------------------------------------------
docs/building.md | 32 +++++++++++++++-
docs/coding_guidelines.md | 62 +-----------------------------
docs/how_to_contribute.md | 85 +-----------------------------------------
3 files changed, 33 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/25b4099d/docs/building.md
----------------------------------------------------------------------
diff --git a/docs/building.md b/docs/building.md
index 86810a4..a07b324 100644
--- a/docs/building.md
+++ b/docs/building.md
@@ -3,7 +3,7 @@ title: "Build Flink"
---
-In order to build Flink, you need the source code. Either download the source of a release or clone the git repository. In addition to that, you need Maven 3 and a JDK. Note that you can not build Flink with Oracle JDK 6 due to a unresolved bug in the Java compiler. It works well with OpenJDK 6 and all Java 7 and 8 compilers.
+In order to build Flink, you need the source code. Either download the source of a release or clone the git repository. In addition to that, you need Maven 3 and a JDK (Java Development Kit). Note that you can not build Flink with Oracle JDK 6 due to a unresolved bug in the Java compiler. It works well with OpenJDK 6 and all Java 7 and 8 compilers.
To clone from git, enter:
```
@@ -13,6 +13,7 @@ git clone {{ site.FLINK_GITHUB_URL }}
The simplest way of building Flink is by running:
```
+cd incubator-flink
mvn clean package -DskipTests
```
@@ -23,7 +24,8 @@ This instructs Maven (`mvn`) to first remove all existing builds (`clean`) and t
## Build Flink for a specific Hadoop Version
This section covers building Flink for a specific Hadoop version. Most users do not need to do this manually.
-The problem is that Flink uses HDFS and YARN which are both from Apache Hadoop. There exist many different builds of Hadoop (from both the upstream project and the different Hadoop distributions). Typically errors arise with the RPC services. An error could look like this:
+
+The problem is that Flink uses HDFS and YARN which are both dependencies from Apache Hadoop. There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If a user is using a wrong combination of versions, exceptions like this one occur:
```
ERROR: The job was not successfully submitted to the nephele job manager:
@@ -32,6 +34,32 @@ ERROR: The job was not successfully submitted to the nephele job manager:
Protocol message contained an invalid tag (zero).; Host Details :
```
+There are two main versions of Hadoop that we need to differentiate:
+- Hadoop 1, with all versions starting with zero or one, like 0.20, 0.23 or 1.2.1.
+- Hadoop 2 with all versions including or higher than 2.2.0.
+The main differentiation between Hadoop 1 and Hadoop 2 is the availability of Hadoop YARN (Hadoops cluster resource manager).
+
+**To build Flink for Hadoop 2**, issue the following command:
+```
+mvn clean package -DskipTests -Dhadoop.profile=2
+```
+
+The `-Dhadoop.profile=2` flag instructs Maven to build Flink with YARN support and the Hadoop 2 HDFS client.
+
+Usually, this flag is sufficient for full support of Flink for Hadoop 2-versions.
+However, you can also **specify a specific Hadoop version to build against**:
+```
+mvn clean package -DskipTests -Dhadoop.profile=2 -Dhadoop.version=2.4.1
+```
+
+
+**To build Flink against a vendor specific Hadoop version**, issue the following command:
+```
+mvn clean package -DskipTests -Pvendor-repos -Dhadoop.version=2.2.0-cdh5.0.0-beta-2
+```
+
+The `-Pvendor-repos` activates a Maven [build profile](http://maven.apache.org/guides/introduction/introduction-to-profiles.html) that includes the repositories of popular Hadoop vendors such as Cloudera, Hortonworks, or MapR.
+
### Background
The builds with Maven are controlled by [properties](http://maven.apache.org/pom.html#Properties) and <a href="http://maven.apache.org/guides/introduction/introduction-to-profiles.html">build profiles</a>.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/25b4099d/docs/coding_guidelines.md
----------------------------------------------------------------------
diff --git a/docs/coding_guidelines.md b/docs/coding_guidelines.md
index 6ab5475..8d96b37 100644
--- a/docs/coding_guidelines.md
+++ b/docs/coding_guidelines.md
@@ -2,64 +2,4 @@
title: "Coding Guidelines"
---
-These are the coding and style guidelines that we follow in the Flink project.
-
-## Guidelines for pull requests and patches
-
-- A pull request should relate to a JIRA issue; create an issue if none exists for the change you want to make. The latest commit message should reference that issue. An example commit message would be "[FLINK-633] Fix NullPointerException for empty UDF parameters". That way, the pull request automatically gives a description of what it does, for example what bug does it fix in what way?
-
-- We consider pull requests as requests to merge the referenced code *as is* into the current *stable* master branch. Therefore, a pull request should not be "work in progress". Open a pull request if you are confident that it can be merged into the current master branch without problems. If you rather want comments on your code, post a link to your working branch.
-
-- Please do not combine various unrelated changes in a single pull request. Rather, open multiple individual pull requests. This ensures that pull requests are *topic related*, can be merged more easily, and typically result in topic-specific merge conflicts only.
-
-- Any pull request where the tests do not pass or which does not compile will not undergo any further review. We recommend to connect your private GitHub accounts with [Travis CI](http://travis-ci.org/) (like the Flink GitHub repository). Travis will run tests for all tested environments whenever you push something into *your* Github repository.
-
-- Please keep reformatting of source files to a minimum. Diffs become unreadable if you (or your IDE automatically) remove or replace whitespaces, reformat code, or comments. Also, other patches that affect the same files become un-mergeable. Please configure your IDE such that code is not automatically reformatted. Pull requests with excessive or unnecessary code reformatting might be rejected.
-
-- All new features need to be backed by tests, *strictly*. It is very easy that a later merge accidentally throws out a feature or breaks it. This will not be caught if the feature is not guarded by tests. Anything not covered by a test is considered cosmetic.
-
-- Before opening a pull request follow this checklist:
- - Rebase onto the latest version of the master branch
- - Clean up your commits, i.e., squash them in a reasonable way and give meaningful commit messages
- - Run *all* tests either locally with ```mvn clean verify``` or use Travis CI to check the build
-
-- When you get comments on the pull request asking for changes, append commits for these changes. *Do not rebase and squash them.* It allows people to review the cleanup work independently. Otherwise reviewers have to go through the entire set of diffs again.
-
-- Public methods and classes that are part of the user-facing API need to have JavaDocs. Please write meaningful docs. Good docs are concise and informative.
-
-- Give meaningful exception messages. Try to imagine why an exception could be thrown (what a user did wrong) and give a message that will help a user to resolve the problem.
-
-- Follow the checkstyle rules (see below). The checkstyle plugin verifies these rules when you build the code. If your code does not follow the checkstyle rules, Maven will not compile it and consequently the build will fail.
-
-
-## Coding Style Guidelines
-
-- Make sure you have Apache License headers in your files. The RAT plugin is checking for that when you build the code.
-
-- We are using tabs for indentation, not spaces. We are not religious there, it just happened to be the way that we started with tabs, and it is important to not mix them (merge/diff conflicts).
-
-- All statements after `if`, `for`, `while`, `do`, ... must always be encapsulated in a block with curly braces (even if the block contains one statement):
-```
-for (...) {
- ...
-}
-```
-If you are wondering why, recall the famous [*goto bug*](https://www.imperialviolet.org/2014/02/22/applebug.html) in Apple's SSL library.
-
-
-- Do not use wildcard imports in the core files. They can cause problems when adding to the code and in some cases even during refactoring. Exceptions are the Tuple classes, Tuple-related utilities, and Flink user programs, when importing operators/functions. Tests are a special case of the user programs.
-
-- Remove all unused imports.
-
-- Do not use raw generic types, unless strictly necessary (sometime necessary for signature matches, arrays).
-
-- Add annotations to suppress warnings, if they cannot be avoided (such as "unchecked", or "serial")
-
-- Do not add "non-javadoc" comments.
-
-- Add comments to your code. What is it doing? Add JavaDocs or inherit them by not adding any comments to the methods. Do not automatically generate comments and avoid unnecessary comments like
-```
-i++; // increment by one
-```
-
-
+The coding guidelines are now located [on the project website](http://flink.incubator.apache.org/coding_guidelines.html).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/25b4099d/docs/how_to_contribute.md
----------------------------------------------------------------------
diff --git a/docs/how_to_contribute.md b/docs/how_to_contribute.md
index 4b532d9..9c28505 100644
--- a/docs/how_to_contribute.md
+++ b/docs/how_to_contribute.md
@@ -1,86 +1,5 @@
---
-title: "Contribute"
+title: "How to contribute"
---
-The Flink project welcomes all sorts contributions in the form of code (improvements, features, bugfixes), documentation, tests, and community participation (discussions & questions).
-
-
-
-## Easy Issues for Starters
-
-We maintain all known issues and feature drafts in the [Flink project JIRA](https://issues.apache.org/jira/browse/FLINK-989?jql=project%20%3D%20FLINK).
-
-We also try to maintain a [list of simple "starter issues"](https://issues.apache.org/jira/browse/FLINK-933?jql=project%20%3D%20FLINK%20AND%20labels%20%3D%20starter) that we believe are good tasks for new contributors. Those tasks are meant to allow people to get into the project and become familiar with the process of contributing. Feel free to ask questions about issues that you would be interested in working on.
-
-In addition, you can find a list of ideas for projects and improvements in the [projects wiki page](http://the/wiki/url).
-
-
-
-## Contributing Code & Documentation
-
-This section gives you a brief introduction in how to contribute code and documentation to Flink. We maintain both the code and the documentation in the same repository, so the process is essentially the same for both. We use [git](http://git-scm.com/) for the code and documentation version control.
-
-The Flink project accepts code contributions though the [GitHub Mirror](https://github.com/apache/incubator-flink), in the form of [Pull Requests](https://help.github.com/articles/using-pull-requests). Pull requests are basically a simpler way of offering a patch, by providing a pointer to a code branch that contains the change.
-
-
-### Setting up the Infrastructure and Creating a Pull Request
-
-1. The first step is to create yourself a copy of the Flink code base. We suggest to fork the [Flink GitHub Mirror Repository](https://github.com/apache/incubator-flink) into your own [GitHub](https://github.com) account. You need to register on GitHub for that, if you have no account so far.
-
-2. Next, clone your repository fork to your local machine.
-```
-git clone https://github.com/<your-user-name>/incubator-flink.git
-```
-
-3. It is typically helpful to switch to a *topic branch* for the changes. To create a dedicated branch based on the current master, use the following command:
-```
-git checkout -b myBranch master
-```
-
-4. Now you can create your changes, compile the code, and validate the changes. Here are some pointers on how to [set up the Eclipse IDE for development](https://github.com/apache/incubator-flink/#eclipse-setup-and-debugging), and how to [build the code](https://github.com/apache/incubator-flink/#build-flink).
-
-5. After you have finalized your contribution, verify the compliance with the contribution guidelines (see below), and commit them. To make the changes easily mergeable, please rebase them to the latest version of the main repositories master branch. Assuming you created a topic branch (step 3), you can follow this sequence of commands to do that:
-Switch to the master branch, update it to the latest revision, switch back to your topic branch, and rebase it on top of the master branch.
-```
-git checkout master
-git pull https://github.com/apache/incubator-flink.git master
-git checkout myBranch
-git rebase master
-```
-Have a look [here](https://help.github.com/articles/using-git-rebase) for more information about rebasing commits.
-
-
-6. Push the contribution it back into your fork of the Flink repository.
-```
-git push origin myBranch
-```
-Go the website of your repository fork (`https://github.com/<your-user-name>/incubator-flink`) and use the "Create Pull Request" button to start creating a pull request. Make sure that the base fork is `apache/incubator-flink master` and the head fork selects the branch with your changes. Give the pull request a meaningful description and send it.
-
-
-### Verifying the Compliance of your Code
-
-Before sending a patch or pull request, please verify that it complies with the guidelines of the project. While we try to keep the set of guidelines small and easy, it is important to follow those rules in order to guarantee good code quality, to allow efficient reviews, and to allow committers to easily merge your changes.
-
-Please have a look at the [coding guidelines](coding_guidelines.html) for a guide to the format of code and pull requests.
-
-Most important of all, verify that your changes are correct and do not break existing functionality. Run the existing tests by calling `mvn verify` in the root directory of the repository, and make sure that the tests succeed. We encourage every contributor to use a *continuous integration* service that will automatically test the code in your repository whenever you push a change. Flink is pre-configured for [Travis CI](http://docs.travis-ci.com/), which can be easily enabled for your private repository fork (it uses GitHub for authentication, so you so not need an additional account). Simply add the *Travis CI* hook to your repository (*settings --> webhooks & services --> add service*) and enable tests for the "incubator-flink" repository on [Travis](https://travis-ci.org/profile).
-
-When contributing documentation, please review the rendered HTML versions of the documents you changed. You can look at the HTML pages by using the rendering script in preview mode.
-```
-cd docs
-./build_docs.sh -p
-```
-Now, open your browser at `http://localhost:4000` and check out the pages you changed.
-
-
-
-## How to become a committer
-
-There is no strict protocol for becoming a committer. Candidates for new committers are typically people that are active contributors and community members.
-
-Being an active community member means participating on mailing list discussions, helping to answer questions, being respectful towards others, and following the meritocratic principles of community management. Since the "Apache Way" has a strong focus on the project community, this part is very important.
-
-Of course, contributing code to the project is important as well. A good way to start is contributing improvements, new features, or bugfixes. You need to show that you take responsibility for the code that you contribute, add tests/documentation, and help maintaining it.
-
-Finally, candidates for new committers are suggested by current committers, mentors, or PMC members, and voted upon by the PMC.
-
+The "How to contribute"-guide is now located [on the project website](http://flink.incubator.apache.org/how-to-contribute.html).
\ No newline at end of file
[39/39] git commit: Extended release script,
update nightly build script
Posted by rm...@apache.org.
Extended release script, update nightly build script
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/d83c8108
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/d83c8108
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/d83c8108
Branch: refs/heads/travis_test
Commit: d83c81086462d4109ca32accd7d96d29a596998e
Parents: 3f1e220
Author: Robert Metzger <rm...@apache.org>
Authored: Sat Aug 9 14:32:47 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Sat Aug 9 14:38:17 2014 +0200
----------------------------------------------------------------------
tools/create_release_files.sh | 39 +++++++++++++++++++++++++++++---------
tools/deploy_to_maven.sh | 20 ++++++++++++-------
2 files changed, 43 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d83c8108/tools/create_release_files.sh
----------------------------------------------------------------------
diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh
index aa8875d..14bf420 100755
--- a/tools/create_release_files.sh
+++ b/tools/create_release_files.sh
@@ -39,37 +39,50 @@ GIT_PASSWORD=${GIT_PASSWORD:-XXX}
GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
GPG_KEY=${GPG_KEY:-XXX}
GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
+OLD_VERSION=${OLD_VERSION:-0.6-incubating-SNAPSHOT}
RELEASE_VERSION=${NEW_VERSION}
+NEW_VERSION_HADOOP2=$RELEASE_VERSION"-hadoop2" # this is wrong!
USER_NAME=${USER_NAME:-pwendell}
MVN=${MVN:-mvn}
+sonatype_user=${sonatype_user:-rmetzger}
+sonatype_pw=${sonatype_pw:-XXX}
set -e
# create source package
-git clone https://github.com/rmetzger/incubator-flink.git flink
+git clone https://github.com/apache/incubator-flink.git flink
cd flink
git checkout -b release-0.6 origin/release-0.6
rm .gitignore
rm .travis.yml
rm deploysettings.xml
rm CHANGELOG
-rm -rf .git
+#rm -rf .git
+
+#find . -name 'pom.xml' -type f -exec sed -i 's#<version>$OLD_VERSION</version>#<version>$NEW_VERSION</version>#' {} \;
+find . -name 'pom.xml' -type f -exec sed -i 's#<version>'$OLD_VERSION'</version>#<version>'$NEW_VERSION'</version>#' {} \;
+git commit --author="Robert Metzger <rm...@apache.org>" -am "Commit for release $RELEASE_VERSION"
+RELEASE_HASH=`git rev-parse HEAD`
+echo "Echo created release hash $RELEASE_HASH"
+
cd ..
+echo "Creating source package"
cp -r flink flink-$RELEASE_VERSION
-tar cvzf flink-$RELEASE_VERSION.tgz flink-$RELEASE_VERSION
-echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --armour --output flink-$RELEASE_VERSION.tgz.asc \
- --detach-sig flink-$RELEASE_VERSION.tgz
-echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md MD5 flink-$RELEASE_VERSION.tgz > \
- flink-$RELEASE_VERSION.tgz.md5
-echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md SHA512 flink-$RELEASE_VERSION.tgz > \
- flink-$RELEASE_VERSION.tgz.sha
+tar cvzf flink-${RELEASE_VERSION}-src.tgz --exclude .git flink-$RELEASE_VERSION
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --armour --output flink-$RELEASE_VERSION-src.tgz.asc \
+ --detach-sig flink-$RELEASE_VERSION-src.tgz
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md MD5 flink-$RELEASE_VERSION-src.tgz > \
+ flink-$RELEASE_VERSION-src.tgz.md5
+echo $GPG_PASSPHRASE | gpg --batch --default-key $GPG_KEY --passphrase-fd 0 --print-md SHA512 flink-$RELEASE_VERSION-src.tgz > \
+ flink-$RELEASE_VERSION-src.tgz.sha
rm -rf flink-$RELEASE_VERSION
make_binary_release() {
+ echo "Creating binary release name: $NAME, flags: $FLAGS"
NAME=$1
FLAGS=$2
cp -r flink flink-$RELEASE_VERSION-bin-$NAME
@@ -132,3 +145,11 @@ ssh $USER_NAME@people.apache.org \
scp flink-* \
$USER_NAME@people.apache.org:/home/$USER_NAME/public_html/$folder/
+echo "Deploying to repository.apache.org"
+
+cd flink
+cp ../../deploysettings.xml .
+$MVN clean deploy -Prelease --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE
+./tools/generate_specific_pom.sh $NEW_VERSION $NEW_VERSION_HADOOP2 pom.xml
+sleep 4
+$MVN clean deploy -Prelease --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d83c8108/tools/deploy_to_maven.sh
----------------------------------------------------------------------
diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh
index 78d1a32..03be4c1 100755
--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -54,7 +54,7 @@ function getVersion() {
# this will take a while
CURRENT_FLINK_VERSION=`getVersion`
if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
- CURRENT_FLINK_VERSION_YARN=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop2-SNAPSHOT}
+ CURRENT_FLINK_VERSION_YARN=${CURRENT_FLINK_VERSION/-incubating-SNAPSHOT/-hadoop2-incubating-SNAPSHOT}
else
CURRENT_FLINK_VERSION_YARN="$CURRENT_FLINK_VERSION-hadoop2"
fi
@@ -106,7 +106,7 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
fi
#
- # Deploy binaries to DOPA
+ # Deploy binaries to S3
# The TRAVIS_JOB_NUMBER here is kinda hacked.
# Currently, there are Builds 1-6. Build 1 is deploying to maven sonatype
# Build 2 has no special meaning, it is the openjdk7, hadoop 1.2.1 build
@@ -121,23 +121,29 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
#./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_YARN pom.xml
#mvn -B -DskipTests clean install
CURRENT_FLINK_VERSION=$CURRENT_FLINK_VERSION_YARN
- YARN_ARCHIVE="flink-dist/target/*yarn.tar.gz"
+ YARN_ARCHIVE="flink-dist/target/flink-*-bin/*yarn.tgz"
fi
if [[ $TRAVIS_JOB_NUMBER == *3 ]] || [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
# cd flink-dist
# mvn -B -DskipTests -Pdebian-package package
# cd ..
echo "Uploading build to amazon s3. Job Number: $TRAVIS_JOB_NUMBER"
- mkdir flink
- cp -r flink-dist/target/flink-dist-*-bin/flink*/* flink/
- tar -czf flink-$CURRENT_FLINK_VERSION.tgz flink
+ #mkdir flink-$CURRENT_FLINK_VERSION
+ #cp -r flink-dist/target/flink-*-bin/flink*/* flink-$CURRENT_FLINK_VERSION/
+ #tar -czf flink-$CURRENT_FLINK_VERSION-bin.tgz flink-$CURRENT_FLINK_VERSION
# upload the two in parallel
if [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
# move to current dir
mv $YARN_ARCHIVE .
- travis-artifacts upload --path *yarn.tar.gz --target-path /
+ travis-artifacts upload --path *yarn.tgz --target-path /
fi
+
+ mv flink-dist/target/flink-*-bin/flink*/flink-*.tgz .
+ echo "here, we have"
+ ls -lisah flink-dist/target/flink-*-bin/flink*/
+ echo "and really here"
+ ls -lisah
travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION.tgz --target-path /
fi
[30/39] git commit: [FLINK-1023] Switch group-at-a-time function to
java.lang.Iterable (from java.util.Iterator) Iterables over transient data
throw an TraversableOnceException when iterated over again.
Posted by rm...@apache.org.
[FLINK-1023] Switch group-at-a-time function to java.lang.Iterable (from java.util.Iterator)
Iterables over transient data throw an TraversableOnceException when iterated over again.
This closes #84
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/72d7b862
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/72d7b862
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/72d7b862
Branch: refs/heads/travis_test
Commit: 72d7b86274c33d1570ffb22b1fca2081c15d753c
Parents: 934e4e0
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Jul 29 15:58:44 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Aug 1 02:33:49 2014 +0200
----------------------------------------------------------------------
.../api/avro/AvroWithEmptyArrayITCase.java | 7 +-
.../mapred/record/example/WordCount.java | 2 +-
.../example/WordCountWithOutputFormat.java | 2 +-
.../spargel/java/VertexCentricIteration.java | 42 ++--
.../spargel/java/record/SpargelIteration.java | 2 +-
.../compiler/CoGroupSolutionSetFirstTest.java | 7 +-
.../compiler/GroupReduceCompilationTest.java | 15 +-
.../flink/compiler/IterationsCompilerTest.java | 5 +-
.../WorksetIterationsJavaApiCompilerTest.java | 10 +-
.../testfunctions/IdentityGroupReducer.java | 8 +-
.../testfunctions/Top1GroupReducer.java | 7 +-
.../flink/compiler/util/DummyCoGroupStub.java | 1 -
.../flink/compiler/util/IdentityReduce.java | 1 -
.../api/common/functions/CoGroupFunction.java | 9 +-
.../api/common/functions/CombineFunction.java | 3 +-
.../common/functions/FlatCombineFunction.java | 3 +-
.../common/functions/GroupReduceFunction.java | 3 +-
.../flink/util/TraversableOnceException.java | 28 +++
.../example/java/graph/EnumTrianglesBasic.java | 10 +-
.../example/java/graph/EnumTrianglesOpt.java | 12 +-
.../flink/example/java/graph/PageRankBasic.java | 6 +-
.../java/graph/TransitiveClosureNaive.java | 6 +-
.../example/java/relational/WebLogAnalysis.java | 10 +-
.../flink/api/java/ExecutionEnvironment.java | 21 ++
.../api/java/functions/GroupReduceIterator.java | 4 +-
.../api/java/functions/RichCoGroupFunction.java | 4 +-
.../java/functions/RichFlatCombineFunction.java | 4 +-
.../java/functions/RichGroupReduceFunction.java | 9 +-
.../api/java/operators/AggregateOperator.java | 3 +-
.../api/java/operators/CoGroupOperator.java | 10 +-
.../api/java/operators/DistinctOperator.java | 6 +-
.../api/java/operators/GroupReduceOperator.java | 5 -
.../PlanUnwrappingCoGroupOperator.java | 44 +---
.../translation/PlanUnwrappingJoinOperator.java | 8 -
.../PlanUnwrappingReduceGroupOperator.java | 14 +-
.../translation/TupleUnwrappingIterator.java | 15 +-
.../operators/translation/WrappingFunction.java | 3 +-
.../java/record/functions/CoGroupFunction.java | 4 +-
.../java/record/functions/ReduceFunction.java | 7 +-
.../java/record/operators/CoGroupOperator.java | 61 ++++-
.../java/record/operators/ReduceOperator.java | 107 ++++++--
.../DeltaIterationTranslationTest.java | 4 +-
.../record/CoGroupWrappingFunctionTest.java | 221 +++++++++++++++++
.../java/record/ReduceWrappingFunctionTest.java | 246 +++++++++++++++++++
.../java/type/extractor/TypeExtractorTest.java | 5 +-
.../javaApiOperators/lambdas/CoGroupITCase.java | 11 +-
.../lambdas/GroupReduceITCase.java | 8 +-
.../CoGroupWithSolutionSetFirstDriver.java | 6 +-
.../CoGroupWithSolutionSetSecondDriver.java | 5 +-
.../runtime/operators/RegularPactTask.java | 1 -
.../sort/CombiningUnilateralSortMerger.java | 16 +-
.../operators/sort/FixedLengthRecordSorter.java | 2 +-
.../runtime/operators/sort/InMemorySorter.java | 2 +-
.../runtime/operators/sort/MergeIterator.java | 17 +-
.../operators/sort/MergeMatchIterator.java | 1 -
.../sort/SortMergeCoGroupIterator.java | 20 +-
.../operators/util/CoGroupTaskIterator.java | 9 +-
.../flink/runtime/util/EmptyIterator.java | 11 +-
.../flink/runtime/util/KeyGroupedIterator.java | 23 +-
.../util/MutableToRegularIteratorWrapper.java | 16 +-
.../runtime/util/SingleElementIterator.java | 7 +-
.../runtime/operators/CachedMatchTaskTest.java | 18 +-
.../operators/CoGroupTaskExternalITCase.java | 19 +-
.../runtime/operators/CoGroupTaskTest.java | 30 +--
.../runtime/operators/CombineTaskTest.java | 35 ++-
.../flink/runtime/operators/MapTaskTest.java | 1 +
.../operators/ReduceTaskExternalITCase.java | 35 ++-
.../flink/runtime/operators/ReduceTaskTest.java | 50 ++--
.../operators/chaining/ChainTaskTest.java | 15 +-
.../drivers/AllGroupReduceDriverTest.java | 16 +-
.../drivers/GroupReduceDriverTest.java | 16 +-
.../CombiningUnilateralSortMergerITCase.java | 24 +-
.../sort/SortMergeCoGroupIteratorITCase.java | 4 +-
.../runtime/util/KeyGroupedIteratorTest.java | 43 +++-
.../api/scala/functions/ReduceFunction.scala | 4 +-
.../api/scala/operators/CoGroupOperator.scala | 7 +-
.../api/scala/operators/ReduceOperator.scala | 5 +-
.../test/accumulators/AccumulatorITCase.java | 1 +
.../AccumulatorIterativeITCase.java | 8 +-
.../KMeansIterativeNepheleITCase.java | 3 +-
.../BulkIterationWithAllReducerITCase.java | 8 +-
.../CoGroupConnectedComponentsITCase.java | 4 +-
.../CoGroupConnectedComponentsSecondITCase.java | 12 +-
.../DependencyConnectedComponentsITCase.java | 30 +--
...IterationTerminationWithTerminationTail.java | 1 -
.../IterationTerminationWithTwoTails.java | 2 +-
.../IterationWithAllReducerITCase.java | 1 -
.../iterative/IterationWithChainingITCase.java | 118 +++++----
.../iterative/IterationWithUnionITCase.java | 81 ++----
...nentsWithParametrizableAggregatorITCase.java | 35 ++-
...entsWithParametrizableConvergenceITCase.java | 20 +-
.../ConnectedComponentsNepheleITCase.java | 5 +-
.../IterationWithChainingNepheleITCase.java | 11 +-
.../CustomCompensatableDotProductCoGroup.java | 11 +-
.../CustomCompensatingMap.java | 1 +
.../CustomRankCombiner.java | 6 +-
.../CompensatableDotProductCoGroup.java | 4 +-
.../test/javaApiOperators/CoGroupITCase.java | 94 +++----
.../javaApiOperators/GroupReduceITCase.java | 123 ++++------
.../flink/test/operators/CoGroupITCase.java | 29 +--
.../flink/test/operators/ReduceITCase.java | 23 +-
.../recordJobTests/GroupOrderReduceITCase.java | 1 -
.../graph/WorksetConnectedComponents.java | 2 +-
103 files changed, 1295 insertions(+), 820 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroWithEmptyArrayITCase.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroWithEmptyArrayITCase.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroWithEmptyArrayITCase.java
index ea9edff..89db1fa 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroWithEmptyArrayITCase.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/avro/AvroWithEmptyArrayITCase.java
@@ -194,15 +194,14 @@ public class AvroWithEmptyArrayITCase extends RecordAPITestBase {
private static final long serialVersionUID = 1L;
@Override
- public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out)
- throws Exception {
+ public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) {
Record r1 = null;
- if (records1.hasNext()) {
+ while (records1.hasNext()) {
r1 = records1.next();
}
Record r2 = null;
- if (records2.hasNext()) {
+ while (records2.hasNext()) {
r2 = records2.next();
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
index 25caf0c..88b1892 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCount.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.hadoopcompatibility.mapred.record.example;
import java.io.Serializable;
@@ -103,6 +102,7 @@ public class WordCount implements Program, ProgramDescription {
public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
Record element = null;
int sum = 0;
+
while (records.hasNext()) {
element = records.next();
int cnt = element.getField(1, IntValue.class).getValue();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
index a3cd3d5..8aaf8e5 100644
--- a/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
+++ b/flink-addons/flink-hadoop-compatibility/src/main/java/org/apache/flink/hadoopcompatibility/mapred/record/example/WordCountWithOutputFormat.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.hadoopcompatibility.mapred.record.example;
import java.io.Serializable;
@@ -101,6 +100,7 @@ public class WordCountWithOutputFormat implements Program, ProgramDescription {
public void reduce(Iterator<Record> records, Collector<Record> out) throws Exception {
Record element = null;
int sum = 0;
+
while (records.hasNext()) {
element = records.next();
int cnt = element.getField(1, IntValue.class).getValue();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
index 65be2f8..8f9149c 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java
@@ -413,24 +413,28 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
}
@Override
- public void coGroup(Iterator<Tuple2<VertexKey, Message>> messages, Iterator<Tuple2<VertexKey, VertexValue>> vertex,
+ public void coGroup(Iterable<Tuple2<VertexKey, Message>> messages, Iterable<Tuple2<VertexKey, VertexValue>> vertex,
Collector<Tuple2<VertexKey, VertexValue>> out)
throws Exception
{
- if (vertex.hasNext()) {
- Tuple2<VertexKey, VertexValue> vertexState = vertex.next();
+ final Iterator<Tuple2<VertexKey, VertexValue>> vertexIter = vertex.iterator();
+
+ if (vertexIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> vertexState = vertexIter.next();
@SuppressWarnings("unchecked")
- Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages;
+ Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages.iterator();
messageIter.setSource(downcastIter);
vertexUpdateFunction.setOutput(vertexState, out);
vertexUpdateFunction.updateVertex(vertexState.f0, vertexState.f1, messageIter);
- } else {
- if (messages.hasNext()) {
+ }
+ else {
+ final Iterator<Tuple2<VertexKey, Message>> messageIter = messages.iterator();
+ if (messageIter.hasNext()) {
String message = "Target vertex does not exist!.";
try {
- Tuple2<VertexKey, Message> next = messages.next();
+ Tuple2<VertexKey, Message> next = messageIter.next();
message = "Target vertex '" + next.f0 + "' does not exist!.";
} catch (Throwable t) {}
throw new Exception(message);
@@ -481,13 +485,15 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
}
@Override
- public void coGroup(Iterator<Tuple2<VertexKey, VertexKey>> edges,
- Iterator<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
+ public void coGroup(Iterable<Tuple2<VertexKey, VertexKey>> edges,
+ Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
throws Exception
{
- if (state.hasNext()) {
- Tuple2<VertexKey, VertexValue> newVertexState = state.next();
- messagingFunction.set((Iterator<?>) edges, out);
+ final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
+
+ if (stateIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
+ messagingFunction.set((Iterator<?>) edges.iterator(), out);
messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
}
}
@@ -534,13 +540,15 @@ public class VertexCentricIteration<VertexKey extends Comparable<VertexKey>, Ver
}
@Override
- public void coGroup(Iterator<Tuple3<VertexKey, VertexKey, EdgeValue>> edges,
- Iterator<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
+ public void coGroup(Iterable<Tuple3<VertexKey, VertexKey, EdgeValue>> edges,
+ Iterable<Tuple2<VertexKey, VertexValue>> state, Collector<Tuple2<VertexKey, Message>> out)
throws Exception
{
- if (state.hasNext()) {
- Tuple2<VertexKey, VertexValue> newVertexState = state.next();
- messagingFunction.set((Iterator<?>) edges, out);
+ final Iterator<Tuple2<VertexKey, VertexValue>> stateIter = state.iterator();
+
+ if (stateIter.hasNext()) {
+ Tuple2<VertexKey, VertexValue> newVertexState = stateIter.next();
+ messagingFunction.set((Iterator<?>) edges.iterator(), out);
messagingFunction.sendMessages(newVertexState.f0, newVertexState.f1);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
index 3a58afc..96bc799 100644
--- a/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
+++ b/flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/record/SpargelIteration.java
@@ -161,7 +161,7 @@ public class SpargelIteration {
@Override
public void coGroup(Iterator<Record> messages, Iterator<Record> vertex, Collector<Record> out) throws Exception {
-
+
if (vertex.hasNext()) {
Record first = vertex.next();
first.getFieldInto(0, vertexKey);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
index 3624d86..2a4d6d8 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/CoGroupSolutionSetFirstTest.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler;
-import java.util.Iterator;
-
import org.apache.flink.api.java.functions.RichCoGroupFunction;
import org.apache.flink.api.java.functions.RichMapFunction;
import org.junit.Assert;
@@ -30,6 +27,7 @@ import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.compiler.CompilerException;
import org.apache.flink.compiler.plan.Channel;
import org.apache.flink.compiler.plan.DualInputPlanNode;
import org.apache.flink.compiler.plan.OptimizedPlan;
@@ -44,8 +42,7 @@ public class CoGroupSolutionSetFirstTest extends CompilerTestBase {
public static class SimpleCGroup extends RichCoGroupFunction<Tuple1<Integer>, Tuple1<Integer>, Tuple1<Integer>> {
@Override
- public void coGroup(Iterator<Tuple1<Integer>> first, Iterator<Tuple1<Integer>> second, Collector<Tuple1<Integer>> out) throws Exception {
- }
+ public void coGroup(Iterable<Tuple1<Integer>> first, Iterable<Tuple1<Integer>> second, Collector<Tuple1<Integer>> out) {}
}
public static class SimpleMap extends RichMapFunction<Tuple1<Integer>, Tuple1<Integer>> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
index b9cc769..9f63683 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/GroupReduceCompilationTest.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler;
-import java.util.Iterator;
-
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
@@ -51,7 +48,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
data.reduceGroup(new RichGroupReduceFunction<Double, Double>() {
- public void reduce(Iterator<Double> values, Collector<Double> out) {}
+ public void reduce(Iterable<Double> values, Collector<Double> out) {}
}).name("reducer")
.print().name("sink");
@@ -95,7 +92,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
GroupReduceOperator<Long, Long> reduced = data.reduceGroup(new RichGroupReduceFunction<Long, Long>() {
- public void reduce(Iterator<Long> values, Collector<Long> out) {}
+ public void reduce(Iterable<Long> values, Collector<Long> out) {}
}).name("reducer");
reduced.setCombinable(true);
@@ -148,7 +145,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
data
.groupBy(1)
.reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer")
.print().name("sink");
@@ -197,7 +194,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
.groupBy(1)
.reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer");
reduced.setCombinable(true);
@@ -256,7 +253,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
public String getKey(Tuple2<String, Double> value) { return value.f0; }
})
.reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer")
.print().name("sink");
@@ -314,7 +311,7 @@ public class GroupReduceCompilationTest extends CompilerTestBase implements java
public String getKey(Tuple2<String, Double> value) { return value.f0; }
})
.reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
- public void reduce(Iterator<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
+ public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
}).name("reducer");
reduced.setCombinable(true);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
index 8fc4324..828a635 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/IterationsCompilerTest.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler;
import static org.junit.Assert.*;
@@ -24,8 +23,6 @@ import static org.junit.Assert.*;
import org.apache.flink.api.java.functions.RichFlatMapFunction;
import org.junit.Test;
-import java.util.Iterator;
-
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
@@ -287,7 +284,7 @@ public class IterationsCompilerTest extends CompilerTestBase {
public static final class Reduce101 extends RichGroupReduceFunction<Tuple1<Long>, Tuple1<Long>> {
@Override
- public void reduce(Iterator<Tuple1<Long>> values, Collector<Tuple1<Long>> out) {}
+ public void reduce(Iterable<Tuple1<Long>> values, Collector<Tuple1<Long>> out) {}
}
@ConstantFields("0")
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
index 64a4791..62d5d9b 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
@@ -16,17 +16,15 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.util.Iterator;
-
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
@@ -214,7 +212,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
iter.getWorkset().join(invariantInput)
.where(1, 2)
.equalTo(1, 2)
- .with(new RichJoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
+ .with(new JoinFunction<Tuple3<Long,Long,Long>, Tuple3<Long, Long, Long>, Tuple3<Long,Long,Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return first;
}
@@ -224,7 +222,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
result.join(iter.getSolutionSet())
.where(1, 0)
.equalTo(0, 2)
- .with(new RichJoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
+ .with(new JoinFunction<Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>, Tuple3<Long, Long, Long>>() {
public Tuple3<Long, Long, Long> join(Tuple3<Long, Long, Long> first, Tuple3<Long, Long, Long> second) {
return second;
}
@@ -283,7 +281,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
DataSet<Tuple3<Long, Long, Long>> nextWorkset = joinedWithSolutionSet.groupBy(1, 2)
.reduceGroup(new RichGroupReduceFunction<Tuple3<Long,Long,Long>, Tuple3<Long,Long,Long>>() {
- public void reduce(Iterator<Tuple3<Long, Long, Long>> values, Collector<Tuple3<Long, Long, Long>> out) {}
+ public void reduce(Iterable<Tuple3<Long, Long, Long>> values, Collector<Tuple3<Long, Long, Long>> out) {}
})
.name(NEXT_WORKSET_REDUCER_NAME)
.withConstantSet("1->1","2->2","0->0");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
index 42275af..e06e3ba 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/IdentityGroupReducer.java
@@ -16,10 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.testfunctions;
-import java.util.Iterator;
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
@@ -32,9 +30,9 @@ public class IdentityGroupReducer<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<T> values, Collector<T> out) {
- while (values.hasNext()) {
- out.collect(values.next());
+ public void reduce(Iterable<T> values, Collector<T> out) {
+ for (T next : values) {
+ out.collect(next);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
index 3f24e65..b1a0e2d 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/testfunctions/Top1GroupReducer.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.testfunctions;
-import java.util.Iterator;
-
import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.util.Collector;
@@ -32,7 +29,7 @@ public class Top1GroupReducer<T> extends RichGroupReduceFunction<T, T> {
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<T> values, Collector<T> out) {
- out.collect(values.next());
+ public void reduce(Iterable<T> values, Collector<T> out) {
+ out.collect(values.iterator().next());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCoGroupStub.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCoGroupStub.java b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCoGroupStub.java
index 6ef1651..13cd37b 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCoGroupStub.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/util/DummyCoGroupStub.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.util;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-compiler/src/test/java/org/apache/flink/compiler/util/IdentityReduce.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/util/IdentityReduce.java b/flink-compiler/src/test/java/org/apache/flink/compiler/util/IdentityReduce.java
index 3f32423..b78a850 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/util/IdentityReduce.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/util/IdentityReduce.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.compiler.util;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
index cc8fe78..1a0abeb 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CoGroupFunction.java
@@ -19,7 +19,6 @@
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-import java.util.Iterator;
import org.apache.flink.util.Collector;
@@ -41,11 +40,11 @@ import org.apache.flink.util.Collector;
* Some keys may only be contained in one of the two original data sets. In that case, the CoGroup function is invoked
* with in empty input for the side of the data set that did not contain elements with that specific key.
*
- * @param <V1> The data type of the first input data set.
- * @param <V2> The data type of the second input data set.
+ * @param <IN1> The data type of the first input data set.
+ * @param <IN2> The data type of the second input data set.
* @param <O> The data type of the returned elements.
*/
-public interface CoGroupFunction<V1, V2, O> extends Function, Serializable {
+public interface CoGroupFunction<IN1, IN2, O> extends Function, Serializable {
/**
* This method must be implemented to provide a user implementation of a
@@ -59,5 +58,5 @@ public interface CoGroupFunction<V1, V2, O> extends Function, Serializable {
* @throws Exception The function may throw Exceptions, which will cause the program to cancel,
* and may trigger the recovery logic.
*/
- void coGroup(Iterator<V1> first, Iterator<V2> second, Collector<O> out) throws Exception;
+ public void coGroup(Iterable<IN1> first, Iterable<IN2> second, Collector<O> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
index aed7fae..0588526 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/CombineFunction.java
@@ -19,7 +19,6 @@
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-import java.util.Iterator;
/**
* Generic interface used for combine functions ("combiners"). Combiners act as auxiliaries to a {@link GroupReduceFunction}
@@ -45,5 +44,5 @@ public interface CombineFunction<T> extends Function, Serializable {
* @throws Exception The function may throw Exceptions, which will cause the program to cancel,
* and may trigger the recovery logic.
*/
- T combine(Iterator<T> values) throws Exception;
+ T combine(Iterable<T> values) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
index edf1614..bbcbd0a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/FlatCombineFunction.java
@@ -19,7 +19,6 @@
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-import java.util.Iterator;
import org.apache.flink.util.Collector;
@@ -47,5 +46,5 @@ public interface FlatCombineFunction<T> extends Function, Serializable {
* @throws Exception The function may throw Exceptions, which will cause the program to cancel,
* and may trigger the recovery logic.
*/
- void combine(Iterator<T> values, Collector<T> out) throws Exception;
+ void combine(Iterable<T> values, Collector<T> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
index 5fa959c..54cc5c7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/GroupReduceFunction.java
@@ -19,7 +19,6 @@
package org.apache.flink.api.common.functions;
import java.io.Serializable;
-import java.util.Iterator;
import org.apache.flink.util.Collector;
@@ -53,5 +52,5 @@ public interface GroupReduceFunction<T, O> extends Function, Serializable {
* @throws Exception This method may throw exceptions. Throwing an exception will cause the operation
* to fail and may trigger recovery.
*/
- void reduce(Iterator<T> values, Collector<O> out) throws Exception;
+ void reduce(Iterable<T> values, Collector<O> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-core/src/main/java/org/apache/flink/util/TraversableOnceException.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/TraversableOnceException.java b/flink-core/src/main/java/org/apache/flink/util/TraversableOnceException.java
new file mode 100644
index 0000000..a15c31c
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/TraversableOnceException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+public class TraversableOnceException extends RuntimeException {
+
+ private static final long serialVersionUID = 7636881584773577290L;
+
+ public TraversableOnceException() {
+ super("The Iterable can be iterated over only once. Only the first call to 'iterator()' will succeed.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
index 2d794bd..3d68e99 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesBasic.java
@@ -35,7 +35,7 @@ import org.apache.flink.example.java.graph.util.EnumTrianglesDataTypes.Edge;
import org.apache.flink.example.java.graph.util.EnumTrianglesDataTypes.Triad;
/**
- * Triangle enumeration is a preprocessing step to find closely connected parts in graphs.
+ * Triangle enumeration is a pre-processing step to find closely connected parts in graphs.
* A triangle consists of three edges that connect three vertices with each other.
*
* <p>
@@ -154,7 +154,9 @@ public class EnumTrianglesBasic {
private final Triad outTriad = new Triad();
@Override
- public void reduce(Iterator<Edge> edges, Collector<Triad> out) throws Exception {
+ public void reduce(Iterable<Edge> edgesIter, Collector<Triad> out) throws Exception {
+
+ final Iterator<Edge> edges = edgesIter.iterator();
// clear vertex list
vertices.clear();
@@ -165,11 +167,11 @@ public class EnumTrianglesBasic {
vertices.add(firstEdge.getSecondVertex());
// build and emit triads
- while(edges.hasNext()) {
+ while (edges.hasNext()) {
Integer higherVertexId = edges.next().getSecondVertex();
// combine vertex with all previously read vertices
- for(Integer lowerVertexId : vertices) {
+ for (Integer lowerVertexId : vertices) {
outTriad.setSecondVertex(lowerVertexId);
outTriad.setThirdVertex(higherVertexId);
out.collect(outTriad);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
index c0ea26a..efccb59 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/EnumTrianglesOpt.java
@@ -38,7 +38,7 @@ import org.apache.flink.example.java.graph.util.EnumTrianglesDataTypes.EdgeWithD
import org.apache.flink.example.java.graph.util.EnumTrianglesDataTypes.Triad;
/**
- * Triangle enumeration is a preprocessing step to find closely connected parts in graphs.
+ * Triangle enumeration is a pre-processing step to find closely connected parts in graphs.
* A triangle consists of three edges that connect three vertices with each other.
*
* <p>
@@ -166,8 +166,9 @@ public class EnumTrianglesOpt {
final EdgeWithDegrees outputEdge = new EdgeWithDegrees();
@Override
- public void reduce(Iterator<Edge> edges, Collector<EdgeWithDegrees> out) throws Exception {
+ public void reduce(Iterable<Edge> edgesIter, Collector<EdgeWithDegrees> out) {
+ Iterator<Edge> edges = edgesIter.iterator();
otherVertices.clear();
// get first edge
@@ -176,7 +177,7 @@ public class EnumTrianglesOpt {
this.otherVertices.add(edge.getSecondVertex());
// get all other edges (assumes edges are sorted by second vertex)
- while(edges.hasNext()) {
+ while (edges.hasNext()) {
edge = edges.next();
Integer otherVertex = edge.getSecondVertex();
// collect unique vertices
@@ -274,7 +275,8 @@ public class EnumTrianglesOpt {
private final Triad outTriad = new Triad();
@Override
- public void reduce(Iterator<Edge> edges, Collector<Triad> out) throws Exception {
+ public void reduce(Iterable<Edge> edgesIter, Collector<Triad> out) throws Exception {
+ final Iterator<Edge> edges = edgesIter.iterator();
// clear vertex list
vertices.clear();
@@ -285,7 +287,7 @@ public class EnumTrianglesOpt {
vertices.add(firstEdge.getSecondVertex());
// build and emit triads
- while(edges.hasNext()) {
+ while (edges.hasNext()) {
Integer higherVertexId = edges.next().getSecondVertex();
// combine vertex with all previously read vertices
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
index ba9754f..e6a9272 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/PageRankBasic.java
@@ -21,7 +21,6 @@ package org.apache.flink.example.java.graph;
import static org.apache.flink.api.java.aggregation.Aggregations.SUM;
import java.util.ArrayList;
-import java.util.Iterator;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
@@ -163,12 +162,11 @@ public class PageRankBasic {
private final ArrayList<Long> neighbors = new ArrayList<Long>();
@Override
- public void reduce(Iterator<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
+ public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
neighbors.clear();
Long id = 0L;
- while (values.hasNext()) {
- Tuple2<Long, Long> n = values.next();
+ for (Tuple2<Long, Long> n : values) {
id = n.f0;
neighbors.add(n.f1);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
index 22054da..e9ba406 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/graph/TransitiveClosureNaive.java
@@ -25,8 +25,6 @@ import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.IterativeDataSet;
-import java.util.Iterator;
-
import org.apache.flink.api.common.ProgramDescription;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.example.java.graph.util.ConnectedComponentsData;
@@ -70,8 +68,8 @@ public class TransitiveClosureNaive implements ProgramDescription {
.groupBy(0, 1)
.reduceGroup(new GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
@Override
- public void reduce(Iterator<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) throws Exception {
- out.collect(values.next());
+ public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) throws Exception {
+ out.collect(values.iterator().next());
}
});
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
index 3033c0d..9ca6ea9 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/WebLogAnalysis.java
@@ -16,10 +16,8 @@
* limitations under the License.
*/
-
package org.apache.flink.example.java.relational;
-import java.util.Iterator;
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.FilterFunction;
@@ -249,12 +247,12 @@ public class WebLogAnalysis {
* 2: AVG_DURATION
*/
@Override
- public void coGroup(Iterator<Tuple3<Integer, String, Integer>> ranks, Iterator<Tuple1<String>> visits, Collector<Tuple3<Integer, String, Integer>> out) {
+ public void coGroup(Iterable<Tuple3<Integer, String, Integer>> ranks, Iterable<Tuple1<String>> visits, Collector<Tuple3<Integer, String, Integer>> out) {
// Check if there is a entry in the visits relation
- if (!visits.hasNext()) {
- while (ranks.hasNext()) {
+ if (!visits.iterator().hasNext()) {
+ for (Tuple3<Integer, String, Integer> next : ranks) {
// Emit all rank pairs
- out.collect(ranks.next());
+ out.collect(next);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index af0ea52..bcc523d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -33,6 +33,7 @@ import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
+import org.apache.flink.api.common.io.FileInputFormat;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.java.io.CollectionInputFormat;
import org.apache.flink.api.java.io.CsvReader;
@@ -253,6 +254,26 @@ public abstract class ExecutionEnvironment {
public CsvReader readCsvFile(String filePath) {
return new CsvReader(filePath, this);
}
+
+ // ------------------------------------ File Input Format -----------------------------------------
+
+ public <X> DataSource<X> readFile(FileInputFormat<X> inputFormat, String filePath) {
+ if (inputFormat == null) {
+ throw new IllegalArgumentException("InputFormat must not be null.");
+ }
+ if (filePath == null) {
+ throw new IllegalArgumentException("The file path must not be null.");
+ }
+
+ inputFormat.setFilePath(new Path(filePath));
+ try {
+ return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
+ }
+ catch (Exception e) {
+ throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +
+ "Please specify the TypeInformation of the produced type explicitly.");
+ }
+ }
// ----------------------------------- Generic Input Format ---------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
index b363606..087808d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/GroupReduceIterator.java
@@ -28,13 +28,13 @@ public abstract class GroupReduceIterator<IN, OUT> extends RichGroupReduceFuncti
private static final long serialVersionUID = 1L;
- public abstract Iterator<OUT> reduceGroup(Iterator<IN> values) throws Exception;
+ public abstract Iterator<OUT> reduceGroup(Iterable<IN> values) throws Exception;
// -------------------------------------------------------------------------------------------
@Override
- public final void reduce(Iterator<IN> values, Collector<OUT> out) throws Exception {
+ public final void reduce(Iterable<IN> values, Collector<OUT> out) throws Exception {
for (Iterator<OUT> iter = reduceGroup(values); iter.hasNext(); ) {
out.collect(iter.next());
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
index e78c31e..3169622 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichCoGroupFunction.java
@@ -18,8 +18,6 @@
package org.apache.flink.api.java.functions;
-import java.util.Iterator;
-
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.RichFunction;
@@ -40,6 +38,6 @@ public abstract class RichCoGroupFunction<IN1, IN2, OUT> extends AbstractRichFun
private static final long serialVersionUID = 1L;
@Override
- public abstract void coGroup(Iterator<IN1> first, Iterator<IN2> second, Collector<OUT> out) throws Exception;
+ public abstract void coGroup(Iterable<IN1> first, Iterable<IN2> second, Collector<OUT> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
index 97c15ff..a5d45aa 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichFlatCombineFunction.java
@@ -24,8 +24,6 @@ import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.util.Collector;
-import java.util.Iterator;
-
/**
* Rich variant of the {@link FlatCombineFunction}. As a {@link RichFunction}, it gives access to the
* {@link org.apache.flink.api.common.functions.RuntimeContext} and provides setup and teardown methods:
@@ -39,5 +37,5 @@ public abstract class RichFlatCombineFunction<T> extends AbstractRichFunction im
private static final long serialVersionUID = 1L;
@Override
- public abstract void combine(Iterator<T> values, Collector<T> out) throws Exception;
+ public abstract void combine(Iterable<T> values, Collector<T> out) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
index 801f592..9198aeb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/functions/RichGroupReduceFunction.java
@@ -22,7 +22,6 @@ import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-import java.util.Iterator;
import org.apache.flink.api.common.functions.AbstractRichFunction;
import org.apache.flink.api.common.functions.FlatCombineFunction;
@@ -44,7 +43,7 @@ public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunct
private static final long serialVersionUID = 1L;
@Override
- public abstract void reduce(Iterator<IN> values, Collector<OUT> out) throws Exception;
+ public abstract void reduce(Iterable<IN> values, Collector<OUT> out) throws Exception;
/**
* The combine methods pre-reduces elements. It may be called on subsets of the data
@@ -59,7 +58,7 @@ public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunct
* <p>
* Since the reduce function will be called on the result of this method, it is important that this
* method returns the same data type as it consumes. By default, this method only calls the
- * {@link #reduce(Iterator, Collector)} method. If the behavior in the pre-reducing is different
+ * {@link #reduce(Iterable, Collector)} method. If the behavior in the pre-reducing is different
* from the final reduce function (for example because the reduce function changes the data type),
* this method must be overwritten, or the execution will fail.
*
@@ -70,7 +69,7 @@ public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunct
* to fail and may trigger recovery.
*/
@Override
- public void combine(Iterator<IN> values, Collector<IN> out) throws Exception {
+ public void combine(Iterable<IN> values, Collector<IN> out) throws Exception {
@SuppressWarnings("unchecked")
Collector<OUT> c = (Collector<OUT>) out;
reduce(values, c);
@@ -80,7 +79,7 @@ public abstract class RichGroupReduceFunction<IN, OUT> extends AbstractRichFunct
/**
* This annotation can be added to classes that extend {@link RichGroupReduceFunction}, in oder to mark
- * them as "combinable". The system may call the {@link RichGroupReduceFunction#combine(Iterator, Collector)}
+ * them as "combinable". The system may call the {@link RichGroupReduceFunction#combine(Iterable, Collector)}
* method on such functions, to pre-reduce the data before transferring it over the network to
* the actual group reduce operation.
* <p>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
index 80a5fa0..1ceb8c8 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
@@ -271,13 +271,14 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
}
@Override
- public void reduce(Iterator<T> values, Collector<T> out) {
+ public void reduce(Iterable<T> records, Collector<T> out) {
final AggregationFunction<Object>[] aggFunctions = this.aggFunctions;
final int[] fieldPositions = this.fieldPositions;
// aggregators are initialized from before
T current = null;
+ final Iterator<T> values = records.iterator();
while (values.hasNext()) {
current = values.next();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
index 744893b..a890b32 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
@@ -301,7 +301,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
/**
* Intermediate step of a CoGroup transformation. <br/>
* To continue the CoGroup transformation, select the grouping key of the first input {@link DataSet} by calling
- * {@link CoGroupOperatorSets#where()} or {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets#where(KeySelector)}.
+ * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets#where(int, int...)} or {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets#where(KeySelector)}.
*
* @param <I1> The type of the first input DataSet of the CoGroup transformation.
* @param <I2> The type of the second input DataSet of the CoGroup transformation.
@@ -328,7 +328,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @param field0 The first index of the Tuple fields of the first co-grouped DataSets that should be used as key
* @param fields The indexes of the Tuple fields of the first co-grouped DataSets that should be used as keys.
* @return An incomplete CoGroup transformation.
- * Call {@link CoGroupOperatorSetsPredicate#equalTo()} to continue the CoGroup.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(int, int...)} to continue the CoGroup.
*
* @see Tuple
* @see DataSet
@@ -348,7 +348,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @param field0 The first field of the Tuple fields of the first co-grouped DataSets that should be used as key
* @param fields The fields of the first co-grouped DataSets that should be used as keys.
* @return An incomplete CoGroup transformation.
- * Call {@link CoGroupOperatorSetsPredicate#equalTo()} to continue the CoGroup.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(int, int...)} to continue the CoGroup.
*
* @see Tuple
* @see DataSet
@@ -367,7 +367,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
*
* @param keyExtractor The KeySelector function which extracts the key values from the DataSet on which it is grouped.
* @return An incomplete CoGroup transformation.
- * Call {@link CoGroupOperatorSetsPredicate#equalTo()} to continue the CoGroup.
+ * Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(int, int...)} to continue the CoGroup.
*
* @see KeySelector
* @see DataSet
@@ -381,7 +381,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
/**
* Intermediate step of a CoGroup transformation. <br/>
* To continue the CoGroup transformation, select the grouping key of the second input {@link DataSet} by calling
- * {@link CoGroupOperatorSetsPredicate#equalTo(int...)} or {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(KeySelector)}.
+ * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(int, int...)} or {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate#equalTo(KeySelector)}.
*
*/
public final class CoGroupOperatorSetsPredicate {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
index 7646fa0..fd35773 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
@@ -18,8 +18,6 @@
package org.apache.flink.api.java.operators;
-import java.util.Iterator;
-
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.MapFunction;
@@ -148,8 +146,8 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
private static final long serialVersionUID = 1L;
@Override
- public void reduce(Iterator<T> values, Collector<T> out) {
- out.collect(values.next());
+ public void reduce(Iterable<T> values, Collector<T> out) {
+ out.collect(values.iterator().next());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
index e1424ad..7b0ad4d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
@@ -37,7 +37,6 @@ import org.apache.flink.types.TypeInformation;
import org.apache.flink.api.java.DataSet;
-
/**
* This operator represents the application of a "reduceGroup" function on a data set, and the
* result data set produced by the function.
@@ -185,7 +184,6 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
else {
throw new UnsupportedOperationException("Unrecognized key type.");
}
-
}
@@ -215,7 +213,4 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return reducer;
}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
index 89290f0..36bae6b 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingCoGroupOperator.java
@@ -18,8 +18,6 @@
package org.apache.flink.api.java.operators.translation;
-import java.util.Iterator;
-
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
@@ -78,47 +76,11 @@ public class PlanUnwrappingCoGroupOperator<I1, I2, OUT, K>
@Override
- public void coGroup(Iterator<Tuple2<K, I1>> records1, Iterator<Tuple2<K, I2>> records2, Collector<OUT> out) throws Exception {
- iter1.set(records1);
- iter2.set(records2);
+ public void coGroup(Iterable<Tuple2<K, I1>> records1, Iterable<Tuple2<K, I2>> records2, Collector<OUT> out) throws Exception {
+ iter1.set(records1.iterator());
+ iter2.set(records2.iterator());
this.wrappedFunction.coGroup(iter1, iter2, out);
}
}
-
- public static class UnwrappingKeyIterator<K, I1> implements Iterator<I1> {
-
- private Iterator<Tuple2<K, I1>> outerIterator;
- I1 firstValue;
-
- public UnwrappingKeyIterator(Iterator<Tuple2<K, I1>> records1) {
- this.outerIterator = records1;
- this.firstValue = null;
- }
-
- public UnwrappingKeyIterator(Iterator<Tuple2<K, I1>> records1, I1 firstValue ) {
- this.outerIterator = records1;
- this.firstValue = firstValue;
- }
-
- @Override
- public boolean hasNext() {
- return firstValue != null || outerIterator.hasNext();
- }
-
- @Override
- public I1 next() {
- if(firstValue != null) {
- firstValue = null;
- return firstValue;
- }
- return outerIterator.next().getField(1);
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
index 73ea004..efd52d5 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingJoinOperator.java
@@ -68,18 +68,10 @@ public class PlanUnwrappingJoinOperator<I1, I2, OUT, K>
super(wrapped);
}
- //@SuppressWarnings("unchecked")
- //@Override
- //public OUT join(Tuple2<K, I1> value1, Tuple2<K, I2> value2) throws Exception {
- // return wrappedFunction.join((I1)(value1.getField(1)), (I2)(value2.getField(1)));
- //}
-
@SuppressWarnings("unchecked")
@Override
public void join (Tuple2<K, I1> value1, Tuple2<K, I2> value2, Collector<OUT> collector) throws Exception {
wrappedFunction.join ((I1)(value1.getField(1)), (I2)(value2.getField(1)), collector);
}
-
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
index 29eb5ed..7e3f0e5 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/PlanUnwrappingReduceGroupOperator.java
@@ -18,8 +18,6 @@
package org.apache.flink.api.java.operators.translation;
-import java.util.Iterator;
-
import org.apache.flink.api.common.functions.FlatCombineFunction;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -65,14 +63,14 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
@Override
- public void reduce(Iterator<Tuple2<K, IN>> values, Collector<OUT> out) throws Exception {
- iter.set(values);
+ public void reduce(Iterable<Tuple2<K, IN>> values, Collector<OUT> out) throws Exception {
+ iter.set(values.iterator());
this.wrappedFunction.reduce(iter, out);
}
@Override
- public void combine(Iterator<Tuple2<K, IN>> values, Collector<Tuple2<K, IN>> out) throws Exception {
- iter.set(values);
+ public void combine(Iterable<Tuple2<K, IN>> values, Collector<Tuple2<K, IN>> out) throws Exception {
+ iter.set(values.iterator());
coll.set(out);
this.wrappedFunction.combine(iter, coll);
}
@@ -98,8 +96,8 @@ public class PlanUnwrappingReduceGroupOperator<IN, OUT, K> extends GroupReduceOp
@Override
- public void reduce(Iterator<Tuple2<K, IN>> values, Collector<OUT> out) throws Exception {
- iter.set(values);
+ public void reduce(Iterable<Tuple2<K, IN>> values, Collector<OUT> out) throws Exception {
+ iter.set(values.iterator());
this.wrappedFunction.reduce(iter, out);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleUnwrappingIterator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleUnwrappingIterator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleUnwrappingIterator.java
index 1f2c208..c09f3a8 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleUnwrappingIterator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/TupleUnwrappingIterator.java
@@ -21,20 +21,23 @@ package org.apache.flink.api.java.operators.translation;
import java.util.Iterator;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.TraversableOnceException;
/**
* An iterator that reads 2-tuples (key value pairs) and returns only the values (second field).
* The iterator also tracks the keys, as the pairs flow though it.
*/
-public class TupleUnwrappingIterator<T, K> implements Iterator<T>, java.io.Serializable {
+public class TupleUnwrappingIterator<T, K> implements Iterator<T>, Iterable<T>, java.io.Serializable {
private static final long serialVersionUID = 1L;
private K lastKey;
private Iterator<Tuple2<K, T>> iterator;
+ private boolean iteratorAvailable;
public void set(Iterator<Tuple2<K, T>> iterator) {
this.iterator = iterator;
+ this.iteratorAvailable = true;
}
public K getLastKey() {
@@ -57,4 +60,14 @@ public class TupleUnwrappingIterator<T, K> implements Iterator<T>, java.io.Seria
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<T> iterator() {
+ if (iteratorAvailable) {
+ iteratorAvailable = false;
+ return this;
+ } else {
+ throw new TraversableOnceException();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
index 267d879..748e3f3 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/translation/WrappingFunction.java
@@ -42,7 +42,7 @@ public abstract class WrappingFunction<T extends Function> extends AbstractRichF
private static final long serialVersionUID = 1L;
- protected final T wrappedFunction;
+ protected T wrappedFunction;
protected WrappingFunction(T wrappedFunction) {
this.wrappedFunction = wrappedFunction;
@@ -174,6 +174,5 @@ public abstract class WrappingFunction<T extends Function> extends AbstractRichF
public <T extends Value> T getPreviousIterationAggregate(String name) {
return ((IterationRuntimeContext) context).<T>getPreviousIterationAggregate(name);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
index e9b5c25..fc7ef49 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/CoGroupFunction.java
@@ -28,7 +28,7 @@ import org.apache.flink.util.Collector;
/**
* The CoGroupFunction is the base class for functions that are invoked by a {@link org.apache.flink.api.java.record.operators.CoGroupOperator}.
*/
-public abstract class CoGroupFunction extends AbstractRichFunction implements org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record> {
+public abstract class CoGroupFunction extends AbstractRichFunction {
private static final long serialVersionUID = 1L;
@@ -45,7 +45,5 @@ public abstract class CoGroupFunction extends AbstractRichFunction implements or
* runtime catches an exception, it aborts the task and lets the fail-over logic
* decide whether to retry the task execution.
*/
- @Override
public abstract void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) throws Exception;
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
index a1e6369..9848499 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/functions/ReduceFunction.java
@@ -16,14 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.api.java.record.functions;
import java.util.Iterator;
import org.apache.flink.api.common.functions.AbstractRichFunction;
-import org.apache.flink.api.common.functions.FlatCombineFunction;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
@@ -31,7 +28,7 @@ import org.apache.flink.util.Collector;
* The ReduceFunction must be extended to provide a reducer implementation, as invoked by a
* {@link org.apache.flink.api.java.record.operators.ReduceOperator}.
*/
-public abstract class ReduceFunction extends AbstractRichFunction implements GroupReduceFunction<Record, Record>, FlatCombineFunction<Record> {
+public abstract class ReduceFunction extends AbstractRichFunction {
private static final long serialVersionUID = 1L;
@@ -47,7 +44,6 @@ public abstract class ReduceFunction extends AbstractRichFunction implements Gro
* runtime catches an exception, it aborts the reduce task and lets the fail-over logic
* decide whether to retry the reduce execution.
*/
- @Override
public abstract void reduce(Iterator<Record> records, Collector<Record> out) throws Exception;
/**
@@ -71,7 +67,6 @@ public abstract class ReduceFunction extends AbstractRichFunction implements Gro
* runtime catches an exception, it aborts the combine task and lets the fail-over logic
* decide whether to retry the combiner execution.
*/
- @Override
public void combine(Iterator<Record> records, Collector<Record> out) throws Exception {
// to be implemented, if the reducer should use a combiner. Note that the combining method
// is only used, if the stub class is further annotated with the annotation
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/72d7b862/flink-java/src/main/java/org/apache/flink/api/java/record/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/CoGroupOperator.java
index a8cedeb..c958b84 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/record/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/record/operators/CoGroupOperator.java
@@ -16,9 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.api.java.record.operators;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -29,13 +31,15 @@ import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.RecordOperator;
import org.apache.flink.api.common.operators.base.CoGroupOperatorBase;
-import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
+import org.apache.flink.api.java.operators.translation.WrappingFunction;
import org.apache.flink.api.java.record.functions.CoGroupFunction;
import org.apache.flink.api.java.record.functions.FunctionAnnotation;
import org.apache.flink.types.Key;
import org.apache.flink.types.Record;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
/**
* CoGroupOperator that applies a {@link CoGroupFunction} to groups of records sharing
@@ -43,7 +47,7 @@ import org.apache.flink.types.Record;
*
* @see CoGroupFunction
*/
-public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record, CoGroupFunction> implements RecordOperator {
+public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record, org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> implements RecordOperator {
/**
* The types of the keys that the operator groups on.
@@ -61,7 +65,8 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
* @param keyColumn2 The position of the key in the second input's records.
*/
public static Builder builder(CoGroupFunction udf, Class<? extends Key<?>> keyClass, int keyColumn1, int keyColumn2) {
- return new Builder(new UserCodeObjectWrapper<CoGroupFunction>(udf), keyClass, keyColumn1, keyColumn2);
+ WrappingCoGroupFunction wrapper = new WrappingCoGroupFunction(udf);
+ return new Builder(new UserCodeObjectWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>>(wrapper), keyClass, keyColumn1, keyColumn2);
}
/**
@@ -75,7 +80,8 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
public static Builder builder(Class<? extends CoGroupFunction> udf, Class<? extends Key<?>> keyClass,
int keyColumn1, int keyColumn2)
{
- return new Builder(new UserCodeClassWrapper<CoGroupFunction>(udf), keyClass, keyColumn1, keyColumn2);
+ WrappingCoGroupFunction wrapper = new WrappingClassCoGroupFunction(udf);
+ return new Builder(new UserCodeObjectWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>>(wrapper), keyClass, keyColumn1, keyColumn2);
}
/**
@@ -96,7 +102,9 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
setBroadcastVariables(builder.broadcastInputs);
setGroupOrderForInputOne(builder.secondaryOrder1);
setGroupOrderForInputTwo(builder.secondaryOrder2);
- setSemanticProperties(FunctionAnnotation.readDualConstantAnnotations(builder.udf));
+
+ CoGroupFunction function = ((WrappingCoGroupFunction) builder.udf.getUserCodeObject()).getWrappedFunction();
+ setSemanticProperties(FunctionAnnotation.readDualConstantAnnotations(new UserCodeObjectWrapper<CoGroupFunction>(function)));
}
// --------------------------------------------------------------------------------------------
@@ -115,7 +123,7 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
public static class Builder {
/* The required parameters */
- private final UserCodeWrapper<CoGroupFunction> udf;
+ private final UserCodeWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> udf;
private final List<Class<? extends Key<?>>> keyClasses;
private final List<Integer> keyColumns1;
private final List<Integer> keyColumns2;
@@ -136,7 +144,7 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
* @param keyColumn1 The position of the key in the first input's records.
* @param keyColumn2 The position of the key in the second input's records.
*/
- protected Builder(UserCodeWrapper<CoGroupFunction> udf, Class<? extends Key<?>> keyClass,
+ protected Builder(UserCodeWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> udf, Class<? extends Key<?>> keyClass,
int keyColumn1, int keyColumn2)
{
this.udf = udf;
@@ -157,7 +165,7 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
*
* @param udf The {@link CoGroupFunction} implementation for this CoGroup operator.
*/
- protected Builder(UserCodeWrapper<CoGroupFunction> udf) {
+ protected Builder(UserCodeWrapper<org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record>> udf) {
this.udf = udf;
this.keyClasses = new ArrayList<Class<? extends Key<?>>>();
this.keyColumns1 = new ArrayList<Integer>();
@@ -338,4 +346,39 @@ public class CoGroupOperator extends CoGroupOperatorBase<Record, Record, Record,
return new CoGroupOperator(this);
}
}
+
+ // ============================================================================================
+
+ public static class WrappingCoGroupFunction extends WrappingFunction<CoGroupFunction>
+ implements org.apache.flink.api.common.functions.CoGroupFunction<Record, Record, Record> {
+
+ private static final long serialVersionUID = 1L;
+
+ public WrappingCoGroupFunction(CoGroupFunction coGrouper) {
+ super(coGrouper);
+ }
+
+ @Override
+ public void coGroup(Iterable<Record> records1, Iterable<Record> records2, Collector<Record> out) throws Exception {
+ this.wrappedFunction.coGroup(records1.iterator(), records2.iterator(), out);
+ }
+ }
+
+ public static final class WrappingClassCoGroupFunction extends WrappingCoGroupFunction {
+
+ private static final long serialVersionUID = 1L;
+
+ public WrappingClassCoGroupFunction(Class<? extends CoGroupFunction> reducer) {
+ super(InstantiationUtil.instantiate(reducer));
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeObject(wrappedFunction.getClass());
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ Class<?> clazz = (Class<?>) in.readObject();
+ this.wrappedFunction = (CoGroupFunction) InstantiationUtil.instantiate(clazz);
+ }
+ }
}
[18/39] [FLINK-701] Refactor Java API to use SAM interfaces.
Introduce RichFunction stubs for all UDFs.
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index a41874c..e8ee0bb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -31,12 +31,13 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.Validate;
-import org.apache.flink.api.common.functions.GenericCoGrouper;
-import org.apache.flink.api.common.functions.GenericCrosser;
-import org.apache.flink.api.common.functions.GenericFlatMap;
-import org.apache.flink.api.common.functions.GenericGroupReduce;
-import org.apache.flink.api.common.functions.GenericJoiner;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.java.functions.InvalidTypesException;
import org.apache.flink.api.java.functions.KeySelector;
@@ -60,64 +61,75 @@ public class TypeExtractor {
// --------------------------------------------------------------------------------------------
@SuppressWarnings("unchecked")
- public static <IN, OUT> TypeInformation<OUT> getMapReturnTypes(GenericMap<IN, OUT> mapInterface, TypeInformation<IN> inType) {
- validateInputType(GenericMap.class, mapInterface.getClass(), 0, inType);
+ public static <IN, OUT> TypeInformation<OUT> getMapReturnTypes(MapFunction<IN, OUT> mapInterface, TypeInformation<IN> inType) {
+ validateInputType(MapFunction.class, mapInterface.getClass(), 0, inType);
if(mapInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) mapInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericMap.class, mapInterface.getClass(), 1, inType, null);
+ return new TypeExtractor().privateCreateTypeInfo(MapFunction.class, mapInterface.getClass(), 1, inType, null);
}
@SuppressWarnings("unchecked")
- public static <IN, OUT> TypeInformation<OUT> getFlatMapReturnTypes(GenericFlatMap<IN, OUT> flatMapInterface, TypeInformation<IN> inType) {
- validateInputType(GenericFlatMap.class, flatMapInterface.getClass(), 0, inType);
+ public static <IN, OUT> TypeInformation<OUT> getFlatMapReturnTypes(FlatMapFunction<IN, OUT> flatMapInterface, TypeInformation<IN> inType) {
+ validateInputType(FlatMapFunction.class, flatMapInterface.getClass(), 0, inType);
if(flatMapInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) flatMapInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericFlatMap.class, flatMapInterface.getClass(), 1, inType, null);
+ return new TypeExtractor().privateCreateTypeInfo(FlatMapFunction.class, flatMapInterface.getClass(), 1, inType, null);
}
@SuppressWarnings("unchecked")
- public static <IN, OUT> TypeInformation<OUT> getGroupReduceReturnTypes(GenericGroupReduce<IN, OUT> groupReduceInterface,
+ public static <IN, OUT> TypeInformation<OUT> getGroupReduceReturnTypes(GroupReduceFunction<IN, OUT> groupReduceInterface,
TypeInformation<IN> inType) {
- validateInputType(GenericGroupReduce.class, groupReduceInterface.getClass(), 0, inType);
+ validateInputType(GroupReduceFunction.class, groupReduceInterface.getClass(), 0, inType);
if(groupReduceInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) groupReduceInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericGroupReduce.class, groupReduceInterface.getClass(), 1, inType, null);
+ return new TypeExtractor().privateCreateTypeInfo(GroupReduceFunction.class, groupReduceInterface.getClass(), 1, inType, null);
}
@SuppressWarnings("unchecked")
- public static <IN1, IN2, OUT> TypeInformation<OUT> getJoinReturnTypes(GenericJoiner<IN1, IN2, OUT> joinInterface,
+ public static <IN1, IN2, OUT> TypeInformation<OUT> getJoinReturnTypes(FlatJoinFunction<IN1, IN2, OUT> joinInterface,
TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
- validateInputType(GenericJoiner.class, joinInterface.getClass(), 0, in1Type);
- validateInputType(GenericJoiner.class, joinInterface.getClass(), 1, in2Type);
+ validateInputType(FlatJoinFunction.class, joinInterface.getClass(), 0, in1Type);
+ validateInputType(FlatJoinFunction.class, joinInterface.getClass(), 1, in2Type);
if(joinInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) joinInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericJoiner.class, joinInterface.getClass(), 2, in1Type, in2Type);
+ return new TypeExtractor().privateCreateTypeInfo(FlatJoinFunction.class, joinInterface.getClass(), 2, in1Type, in2Type);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <IN1, IN2, OUT> TypeInformation<OUT> getJoinReturnTypes(JoinFunction<IN1, IN2, OUT> joinInterface,
+ TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
+ validateInputType(JoinFunction.class, joinInterface.getClass(), 0, in1Type);
+ validateInputType(JoinFunction.class, joinInterface.getClass(), 1, in2Type);
+ if(joinInterface instanceof ResultTypeQueryable) {
+ return ((ResultTypeQueryable<OUT>) joinInterface).getProducedType();
+ }
+ return new TypeExtractor().privateCreateTypeInfo(JoinFunction.class, joinInterface.getClass(), 2, in1Type, in2Type);
}
@SuppressWarnings("unchecked")
- public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(GenericCoGrouper<IN1, IN2, OUT> coGroupInterface,
+ public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(CoGroupFunction<IN1, IN2, OUT> coGroupInterface,
TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
- validateInputType(GenericCoGrouper.class, coGroupInterface.getClass(), 0, in1Type);
- validateInputType(GenericCoGrouper.class, coGroupInterface.getClass(), 1, in2Type);
+ validateInputType(CoGroupFunction.class, coGroupInterface.getClass(), 0, in1Type);
+ validateInputType(CoGroupFunction.class, coGroupInterface.getClass(), 1, in2Type);
if(coGroupInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) coGroupInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericCoGrouper.class, coGroupInterface.getClass(), 2, in1Type, in2Type);
+ return new TypeExtractor().privateCreateTypeInfo(CoGroupFunction.class, coGroupInterface.getClass(), 2, in1Type, in2Type);
}
@SuppressWarnings("unchecked")
- public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(GenericCrosser<IN1, IN2, OUT> crossInterface,
+ public static <IN1, IN2, OUT> TypeInformation<OUT> getCrossReturnTypes(CrossFunction<IN1, IN2, OUT> crossInterface,
TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type) {
- validateInputType(GenericCrosser.class, crossInterface.getClass(), 0, in1Type);
- validateInputType(GenericCrosser.class, crossInterface.getClass(), 1, in2Type);
+ validateInputType(CrossFunction.class, crossInterface.getClass(), 0, in1Type);
+ validateInputType(CrossFunction.class, crossInterface.getClass(), 1, in2Type);
if(crossInterface instanceof ResultTypeQueryable) {
return ((ResultTypeQueryable<OUT>) crossInterface).getProducedType();
}
- return new TypeExtractor().privateCreateTypeInfo(GenericCrosser.class, crossInterface.getClass(), 2, in1Type, in2Type);
+ return new TypeExtractor().privateCreateTypeInfo(CrossFunction.class, crossInterface.getClass(), 2, in1Type, in2Type);
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
index 8e0abcb..c786345 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
@@ -235,7 +235,7 @@ public final class TupleComparator<T extends Tuple> extends TypeComparator<T> im
try {
for (; i < keyPositions.length; i++) {
int keyPos = keyPositions[i];
- int cmp = comparators[i].compare(first.getField(keyPos), second.getField(keyPos));
+ int cmp = comparators[i].compare((T)first.getField(keyPos), (T)second.getField(keyPos));
if (cmp != 0) {
return cmp;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropertiesTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropertiesTranslationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropertiesTranslationTest.java
index 1159512..474b022 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropertiesTranslationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/functions/SemanticPropertiesTranslationTest.java
@@ -28,8 +28,6 @@ import org.apache.flink.api.common.operators.base.GenericDataSinkBase;
import org.apache.flink.api.common.operators.base.JoinOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.operators.util.FieldSet;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsFirst;
import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsSecond;
@@ -284,7 +282,7 @@ public class SemanticPropertiesTranslationTest {
@ConstantFields("*")
- public static class WildcardConstantMapper<T> extends MapFunction<T, T> {
+ public static class WildcardConstantMapper<T> extends RichMapFunction<T, T> {
@Override
public T map(T value) {
@@ -293,7 +291,7 @@ public class SemanticPropertiesTranslationTest {
}
@ConstantFields("0->0;1->1;2->2")
- public static class IndividualConstantMapper<X, Y, Z> extends MapFunction<Tuple3<X, Y, Z>, Tuple3<X, Y, Z>> {
+ public static class IndividualConstantMapper<X, Y, Z> extends RichMapFunction<Tuple3<X, Y, Z>, Tuple3<X, Y, Z>> {
@Override
public Tuple3<X, Y, Z> map(Tuple3<X, Y, Z> value) {
@@ -302,7 +300,7 @@ public class SemanticPropertiesTranslationTest {
}
@ConstantFields("0")
- public static class ZeroConstantMapper<T> extends MapFunction<T, T> {
+ public static class ZeroConstantMapper<T> extends RichMapFunction<T, T> {
@Override
public T map(T value) {
@@ -312,7 +310,7 @@ public class SemanticPropertiesTranslationTest {
@ConstantFieldsFirst("1 -> 0")
@ConstantFieldsSecond("1 -> 1")
- public static class ForwardingTupleJoin<A, B, C, D> extends JoinFunction<Tuple2<A, B>, Tuple2<C, D>, Tuple2<B, D>> {
+ public static class ForwardingTupleJoin<A, B, C, D> extends RichJoinFunction<Tuple2<A, B>, Tuple2<C, D>, Tuple2<B, D>> {
@Override
public Tuple2<B, D> join(Tuple2<A, B> first, Tuple2<C, D> second) {
@@ -322,7 +320,7 @@ public class SemanticPropertiesTranslationTest {
@ConstantFieldsFirst("0 -> 0")
@ConstantFieldsSecond("0 -> 1")
- public static class ForwardingBasicJoin<A, B> extends JoinFunction<A, B, Tuple2<A, B>> {
+ public static class ForwardingBasicJoin<A, B> extends RichJoinFunction<A, B, Tuple2<A, B>> {
@Override
public Tuple2<A, B> join(A first, B second) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
index db795d9..155bbd1 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/DeltaIterationTranslationTest.java
@@ -35,9 +35,9 @@ import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.DeltaIteration;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.JoinFunction;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichJoinFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.util.Collector;
@@ -128,8 +128,14 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
assertEquals(IdentityMapper.class, worksetMapper.getUserCodeWrapper().getUserCodeClass());
assertEquals(NextWorksetMapper.class, nextWorksetMapper.getUserCodeWrapper().getUserCodeClass());
- assertEquals(SolutionWorksetJoin.class, solutionSetJoin.getUserCodeWrapper().getUserCodeClass());
-
+ if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) {
+ WrappingFunction wf = (WrappingFunction) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
+ assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
+ }
+ else {
+ assertEquals(SolutionWorksetJoin.class, solutionSetJoin.getUserCodeWrapper().getUserCodeClass());
+ }
+
assertEquals(BEFORE_NEXT_WORKSET_MAP, nextWorksetMapper.getName());
assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
@@ -215,21 +221,21 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
// --------------------------------------------------------------------------------------------
- public static class SolutionWorksetJoin extends JoinFunction<Tuple2<Double, String>, Tuple3<Double, Long, String>, Tuple3<Double, Long, String>> {
+ public static class SolutionWorksetJoin extends RichJoinFunction<Tuple2<Double, String>, Tuple3<Double, Long, String>, Tuple3<Double, Long, String>> {
@Override
public Tuple3<Double, Long, String> join(Tuple2<Double, String> first, Tuple3<Double, Long, String> second){
return null;
}
}
- public static class NextWorksetMapper extends MapFunction<Tuple3<Double, Long, String>, Tuple2<Double, String>> {
+ public static class NextWorksetMapper extends RichMapFunction<Tuple3<Double, Long, String>, Tuple2<Double, String>> {
@Override
public Tuple2<Double, String> map(Tuple3<Double, Long, String> value) {
return null;
}
}
- public static class IdentityMapper<T> extends MapFunction<T, T> {
+ public static class IdentityMapper<T> extends RichMapFunction<T, T> {
@Override
public T map(T value) throws Exception {
@@ -237,7 +243,7 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
}
}
- public static class SolutionWorksetCoGroup1 extends CoGroupFunction<Tuple2<Double, String>, Tuple3<Double, Long, String>, Tuple3<Double, Long, String>> {
+ public static class SolutionWorksetCoGroup1 extends RichCoGroupFunction<Tuple2<Double, String>, Tuple3<Double, Long, String>, Tuple3<Double, Long, String>> {
@Override
public void coGroup(Iterator<Tuple2<Double, String>> first, Iterator<Tuple3<Double, Long, String>> second,
@@ -245,7 +251,7 @@ public class DeltaIterationTranslationTest implements java.io.Serializable {
}
}
- public static class SolutionWorksetCoGroup2 extends CoGroupFunction<Tuple3<Double, Long, String>, Tuple2<Double, String>, Tuple3<Double, Long, String>> {
+ public static class SolutionWorksetCoGroup2 extends RichCoGroupFunction<Tuple3<Double, Long, String>, Tuple2<Double, String>, Tuple3<Double, Long, String>> {
@Override
public void coGroup(Iterator<Tuple3<Double, Long, String>> second, Iterator<Tuple2<Double, String>> first,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/ReduceTranslationTests.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/ReduceTranslationTests.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/ReduceTranslationTests.java
index 9f6a6d8..8e457ce 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/ReduceTranslationTests.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/translation/ReduceTranslationTests.java
@@ -27,9 +27,7 @@ import org.apache.flink.api.common.operators.base.GenericDataSourceBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.operators.base.ReduceOperatorBase;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.ReduceFunction;
-import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
-import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceOperator;
+import org.apache.flink.api.java.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -53,7 +51,7 @@ public class ReduceTranslationTests implements java.io.Serializable {
DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);
- initialData.reduce(new ReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
+ initialData.reduce(new RichReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
public Tuple3<Double, StringValue, LongValue> reduce(Tuple3<Double, StringValue, LongValue> value1, Tuple3<Double, StringValue, LongValue> value2) {
return value1;
}
@@ -94,7 +92,7 @@ public class ReduceTranslationTests implements java.io.Serializable {
initialData
.groupBy(2)
- .reduce(new ReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
+ .reduce(new RichReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
public Tuple3<Double, StringValue, LongValue> reduce(Tuple3<Double, StringValue, LongValue> value1, Tuple3<Double, StringValue, LongValue> value2) {
return value1;
}
@@ -141,7 +139,7 @@ public class ReduceTranslationTests implements java.io.Serializable {
return value.f1;
}
})
- .reduce(new ReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
+ .reduce(new RichReduceFunction<Tuple3<Double,StringValue,LongValue>>() {
public Tuple3<Double, StringValue, LongValue> reduce(Tuple3<Double, StringValue, LongValue> value1, Tuple3<Double, StringValue, LongValue> value2) {
return value1;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index b284052..c6ad73d 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -23,16 +23,16 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.flink.api.common.functions.GenericMap;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.java.functions.CoGroupFunction;
-import org.apache.flink.api.java.functions.CrossFunction;
-import org.apache.flink.api.java.functions.FlatMapFunction;
-import org.apache.flink.api.java.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.functions.RichCrossFunction;
+import org.apache.flink.api.java.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.functions.RichGroupReduceFunction;
import org.apache.flink.api.java.functions.InvalidTypesException;
-import org.apache.flink.api.java.functions.JoinFunction;
+import org.apache.flink.api.java.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.functions.MapFunction;
+import org.apache.flink.api.java.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -68,7 +68,7 @@ public class TypeExtractorTest {
@Test
public void testBasicType() {
// use getGroupReduceReturnTypes()
- GroupReduceFunction<?, ?> function = new GroupReduceFunction<Boolean, Boolean>() {
+ RichGroupReduceFunction<?, ?> function = new RichGroupReduceFunction<Boolean, Boolean>() {
private static final long serialVersionUID = 1L;
@Override
@@ -107,7 +107,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testWritableType() {
- MapFunction<?, ?> function = new MapFunction<MyWritable, MyWritable>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<MyWritable, MyWritable>() {
private static final long serialVersionUID = 1L;
@Override
@@ -127,7 +127,7 @@ public class TypeExtractorTest {
@Test
public void testTupleWithBasicTypes() throws Exception {
// use getMapReturnTypes()
- MapFunction<?, ?> function = new MapFunction<Tuple9<Integer, Long, Double, Float, Boolean, String, Character, Short, Byte>, Tuple9<Integer, Long, Double, Float, Boolean, String, Character, Short, Byte>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Tuple9<Integer, Long, Double, Float, Boolean, String, Character, Short, Byte>, Tuple9<Integer, Long, Double, Float, Boolean, String, Character, Short, Byte>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -192,7 +192,7 @@ public class TypeExtractorTest {
@Test
public void testTupleWithTuples() {
// use getFlatMapReturnTypes()
- FlatMapFunction<?, ?> function = new FlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
+ RichFlatMapFunction<?, ?> function = new RichFlatMapFunction<Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>, Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -247,12 +247,12 @@ public class TypeExtractorTest {
@Test
public void testSubclassOfTuple() {
// use getJoinReturnTypes()
- JoinFunction<?, ?, ?> function = new JoinFunction<CustomTuple, String, CustomTuple>() {
+ RichFlatJoinFunction<?, ?, ?> function = new RichFlatJoinFunction<CustomTuple, String, CustomTuple>() {
private static final long serialVersionUID = 1L;
@Override
- public CustomTuple join(CustomTuple first, String second) throws Exception {
- return null;
+ public void join(CustomTuple first, String second, Collector<CustomTuple> out) throws Exception {
+ out.collect(null);
}
};
@@ -295,7 +295,7 @@ public class TypeExtractorTest {
@Test
public void testCustomType() {
// use getCrossReturnTypes()
- CrossFunction<?, ?, ?> function = new CrossFunction<CustomType, Integer, CustomType>() {
+ RichCrossFunction<?, ?, ?> function = new RichCrossFunction<CustomType, Integer, CustomType>() {
private static final long serialVersionUID = 1L;
@Override
@@ -342,7 +342,7 @@ public class TypeExtractorTest {
@Test
public void testTupleWithCustomType() {
// use getMapReturnTypes()
- MapFunction<?, ?> function = new MapFunction<Tuple2<Long, CustomType>, Tuple2<Long, CustomType>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Tuple2<Long, CustomType>, Tuple2<Long, CustomType>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -412,7 +412,7 @@ public class TypeExtractorTest {
@Test
public void testTupleOfValues() {
// use getMapReturnTypes()
- MapFunction<?, ?> function = new MapFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -451,7 +451,7 @@ public class TypeExtractorTest {
@Test
public void testGenericsNotInSuperclass() {
// use getMapReturnTypes()
- MapFunction<?, ?> function = new MapFunction<LongKeyValue<String>, LongKeyValue<String>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<LongKeyValue<String>, LongKeyValue<String>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -494,7 +494,7 @@ public class TypeExtractorTest {
@Test
public void testChainedGenericsNotInSuperclass() {
// use TypeExtractor
- MapFunction<?, ?> function = new MapFunction<ChainedTwo<Integer>, ChainedTwo<Integer>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<ChainedTwo<Integer>, ChainedTwo<Integer>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -536,7 +536,7 @@ public class TypeExtractorTest {
@Test
public void testGenericsInDirectSuperclass() {
// use TypeExtractor
- MapFunction<?, ?> function = new MapFunction<ChainedThree, ChainedThree>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<ChainedThree, ChainedThree>() {
private static final long serialVersionUID = 1L;
@Override
@@ -562,7 +562,7 @@ public class TypeExtractorTest {
@Test
public void testGenericsNotInSuperclassWithNonGenericClassAtEnd() {
// use TypeExtractor
- MapFunction<?, ?> function = new MapFunction<ChainedFour, ChainedFour>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<ChainedFour, ChainedFour>() {
private static final long serialVersionUID = 1L;
@Override
@@ -587,7 +587,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testMissingTupleGenericsException() {
- MapFunction<?, ?> function = new MapFunction<String, Tuple2>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<String, Tuple2>() {
private static final long serialVersionUID = 1L;
@Override
@@ -607,7 +607,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testTupleSupertype() {
- MapFunction<?, ?> function = new MapFunction<String, Tuple>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<String, Tuple>() {
private static final long serialVersionUID = 1L;
@Override
@@ -635,7 +635,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testSameGenericVariable() {
- MapFunction<?, ?> function = new MapFunction<SameTypeVariable<String>, SameTypeVariable<String>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<SameTypeVariable<String>, SameTypeVariable<String>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -667,7 +667,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testNestedTupleGenerics() {
- MapFunction<?, ?> function = new MapFunction<Nested<String, Integer>, Nested<String, Integer>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Nested<String, Integer>, Nested<String, Integer>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -706,7 +706,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testNestedTupleGenerics2() {
- MapFunction<?, ?> function = new MapFunction<Nested2<Boolean>, Nested2<Boolean>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Nested2<Boolean>, Nested2<Boolean>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -746,7 +746,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testFunctionWithMissingGenerics() {
- MapFunction function = new MapFunction() {
+ RichMapFunction function = new RichMapFunction() {
private static final long serialVersionUID = 1L;
@Override
@@ -776,7 +776,7 @@ public class TypeExtractorTest {
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti);
}
- public class IdentityMapper<T> extends MapFunction<T, T> {
+ public class IdentityMapper<T> extends RichMapFunction<T, T> {
private static final long serialVersionUID = 1L;
@Override
@@ -807,7 +807,7 @@ public class TypeExtractorTest {
}
}
- public class IdentityMapper2<T> extends MapFunction<Tuple2<T, String>, T> {
+ public class IdentityMapper2<T> extends RichMapFunction<Tuple2<T, String>, T> {
private static final long serialVersionUID = 1L;
@Override
@@ -843,7 +843,7 @@ public class TypeExtractorTest {
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1));
}
- public class IdentityMapper3<T, V> extends MapFunction<T, V> {
+ public class IdentityMapper3<T, V> extends RichMapFunction<T, V> {
private static final long serialVersionUID = 1L;
@Override
@@ -916,7 +916,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testFunctionWithNoGenericSuperclass() {
- MapFunction<?, ?> function = new Mapper2();
+ RichMapFunction<?, ?> function = new Mapper2();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("String"));
@@ -924,7 +924,7 @@ public class TypeExtractorTest {
Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, ti);
}
- public class OneAppender<T> extends MapFunction<T, Tuple2<T, Integer>> {
+ public class OneAppender<T> extends RichMapFunction<T, Tuple2<T, Integer>> {
private static final long serialVersionUID = 1L;
public Tuple2<T, Integer> map(T value) {
@@ -935,7 +935,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testFunctionDependingPartialOnInput() {
- MapFunction<?, ?> function = new OneAppender<DoubleValue>() {
+ RichMapFunction<?, ?> function = new OneAppender<DoubleValue>() {
private static final long serialVersionUID = 1L;
};
@@ -955,7 +955,7 @@ public class TypeExtractorTest {
@Test
public void testFunctionDependingPartialOnInput2() {
- MapFunction<DoubleValue, ?> function = new OneAppender<DoubleValue>();
+ RichMapFunction<DoubleValue, ?> function = new OneAppender<DoubleValue>();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, new ValueTypeInfo<DoubleValue>(DoubleValue.class));
@@ -971,7 +971,7 @@ public class TypeExtractorTest {
Assert.assertEquals(Integer.class , tti.getTypeAt(1).getTypeClass());
}
- public class FieldDuplicator<T> extends MapFunction<T, Tuple2<T, T>> {
+ public class FieldDuplicator<T> extends RichMapFunction<T, Tuple2<T, T>> {
private static final long serialVersionUID = 1L;
public Tuple2<T, T> map(T value) {
@@ -981,7 +981,7 @@ public class TypeExtractorTest {
@Test
public void testFunctionInputInOutputMultipleTimes() {
- MapFunction<Float, ?> function = new FieldDuplicator<Float>();
+ RichMapFunction<Float, ?> function = new FieldDuplicator<Float>();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, BasicTypeInfo.FLOAT_TYPE_INFO);
@@ -994,7 +994,7 @@ public class TypeExtractorTest {
@Test
public void testFunctionInputInOutputMultipleTimes2() {
- MapFunction<Tuple2<Float, Float>, ?> function = new FieldDuplicator<Tuple2<Float, Float>>();
+ RichMapFunction<Tuple2<Float, Float>, ?> function = new FieldDuplicator<Tuple2<Float, Float>>();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, new TupleTypeInfo<Tuple2<Float, Float>>(
BasicTypeInfo.FLOAT_TYPE_INFO, BasicTypeInfo.FLOAT_TYPE_INFO));
@@ -1023,7 +1023,7 @@ public class TypeExtractorTest {
@Test
public void testAbstractAndInterfaceTypesException() {
- MapFunction<String, ?> function = new MapFunction<String, Testable>() {
+ RichMapFunction<String, ?> function = new RichMapFunction<String, Testable>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1039,7 +1039,7 @@ public class TypeExtractorTest {
// good
}
- MapFunction<String, ?> function2 = new MapFunction<String, AbstractClass>() {
+ RichMapFunction<String, ?> function2 = new RichMapFunction<String, AbstractClass>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1059,7 +1059,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testValueSupertypeException() {
- MapFunction<?, ?> function = new MapFunction<StringValue, Value>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<StringValue, Value>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1080,7 +1080,7 @@ public class TypeExtractorTest {
@Test
public void testBasicArray() {
// use getCoGroupReturnTypes()
- CoGroupFunction<?, ?, ?> function = new CoGroupFunction<String[], String[], String[]>() {
+ RichCoGroupFunction<?, ?, ?> function = new RichCoGroupFunction<String[], String[], String[]>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1107,7 +1107,7 @@ public class TypeExtractorTest {
@Test
public void testBasicArray2() {
- MapFunction<Boolean[], ?> function = new IdentityMapper<Boolean[]>();
+ RichMapFunction<Boolean[], ?> function = new IdentityMapper<Boolean[]>();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO);
@@ -1122,7 +1122,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testCustomArray() {
- MapFunction<?, ?> function = new MapFunction<CustomArrayObject[], CustomArrayObject[]>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<CustomArrayObject[], CustomArrayObject[]>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1140,7 +1140,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTupleArray() {
- MapFunction<?, ?> function = new MapFunction<Tuple2<String, String>[], Tuple2<String, String>[]>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Tuple2<String, String>[], Tuple2<String, String>[]>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1167,7 +1167,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testCustomArrayWithTypeVariable() {
- MapFunction<CustomArrayObject2<Boolean>[], ?> function = new IdentityMapper<CustomArrayObject2<Boolean>[]>();
+ RichMapFunction<CustomArrayObject2<Boolean>[], ?> function = new IdentityMapper<CustomArrayObject2<Boolean>[]>();
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("Tuple1<Boolean>[]"));
@@ -1178,7 +1178,7 @@ public class TypeExtractorTest {
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, tti.getTypeAt(0));
}
- public class GenericArrayClass<T> extends MapFunction<T[], T[]> {
+ public class GenericArrayClass<T> extends RichMapFunction<T[], T[]> {
private static final long serialVersionUID = 1L;
@Override
@@ -1207,7 +1207,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testParamertizedCustomObject() {
- MapFunction<?, ?> function = new MapFunction<MyObject<String>, MyObject<String>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<MyObject<String>, MyObject<String>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1242,7 +1242,7 @@ public class TypeExtractorTest {
@Test
public void testInputMismatchExceptions() {
- MapFunction<?, ?> function = new MapFunction<Tuple2<String, String>, String>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<Tuple2<String, String>, String>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1265,7 +1265,7 @@ public class TypeExtractorTest {
// right
}
- MapFunction<?, ?> function2 = new MapFunction<StringValue, String>() {
+ RichMapFunction<?, ?> function2 = new RichMapFunction<StringValue, String>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1281,7 +1281,7 @@ public class TypeExtractorTest {
// right
}
- MapFunction<?, ?> function3 = new MapFunction<Tuple1<Integer>[], String>() {
+ RichMapFunction<?, ?> function3 = new RichMapFunction<Tuple1<Integer>[], String>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1297,7 +1297,7 @@ public class TypeExtractorTest {
// right
}
- MapFunction<?, ?> function4 = new MapFunction<Writable, String>() {
+ RichMapFunction<?, ?> function4 = new RichMapFunction<Writable, String>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1314,7 +1314,7 @@ public class TypeExtractorTest {
}
}
- public static class DummyFlatMapFunction<A,B,C,D> extends FlatMapFunction<Tuple2<A,B>, Tuple2<C,D>> {
+ public static class DummyFlatMapFunction<A,B,C,D> extends RichFlatMapFunction<Tuple2<A,B>, Tuple2<C,D>> {
private static final long serialVersionUID = 1L;
@Override
@@ -1336,7 +1336,7 @@ public class TypeExtractorTest {
}
}
- public static class MyQueryableMapper<A> extends MapFunction<String, A> implements ResultTypeQueryable<A> {
+ public static class MyQueryableMapper<A> extends RichMapFunction<String, A> implements ResultTypeQueryable<A> {
private static final long serialVersionUID = 1L;
@SuppressWarnings("unchecked")
@@ -1359,7 +1359,7 @@ public class TypeExtractorTest {
@Test
public void testTupleWithPrimitiveArray() {
- MapFunction<Integer, Tuple9<int[],double[],long[],byte[],char[],float[],short[], boolean[], String[]>> function = new MapFunction<Integer, Tuple9<int[],double[],long[],byte[],char[],float[],short[], boolean[], String[]>>() {
+ RichMapFunction<Integer, Tuple9<int[],double[],long[],byte[],char[],float[],short[], boolean[], String[]>> function = new RichMapFunction<Integer, Tuple9<int[],double[],long[],byte[],char[],float[],short[], boolean[], String[]>>() {
private static final long serialVersionUID = 1L;
@Override
@@ -1382,8 +1382,8 @@ public class TypeExtractorTest {
}
@Test
- public void testInterface() {
- GenericMap<String, Boolean> mapInterface = new GenericMap<String, Boolean>() {
+ public void testFunction() {
+ RichMapFunction<String, Boolean> mapInterface = new RichMapFunction<String, Boolean>() {
@Override
public void setRuntimeContext(RuntimeContext t) {
@@ -1392,7 +1392,6 @@ public class TypeExtractorTest {
@Override
public void open(Configuration parameters) throws Exception {
-
}
@Override
@@ -1414,4 +1413,17 @@ public class TypeExtractorTest {
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(mapInterface, BasicTypeInfo.STRING_TYPE_INFO);
Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti);
}
+
+ @Test
+ public void testInterface() {
+ MapFunction<String, Boolean> mapInterface = new MapFunction<String, Boolean>() {
+ @Override
+ public Boolean map(String record) throws Exception {
+ return null;
+ }
+ };
+
+ TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(mapInterface, BasicTypeInfo.STRING_TYPE_INFO);
+ Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-java8-tests/pom.xml b/flink-java8-tests/pom.xml
new file mode 100644
index 0000000..2587776
--- /dev/null
+++ b/flink-java8-tests/pom.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>0.6-incubating-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-java8-tests</artifactId>
+ <name>flink-java8-tests</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.7</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <!-- just define the Java version to be used for compiling and plugins -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version><!--$NO-MVN-MAN-VER$-->
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ <!-- High optimization, no debugging <compilerArgument>-g:none -O</compilerArgument> -->
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <log.level>WARN</log.level>
+ </systemPropertyVariables>
+ <forkMode>once</forkMode>
+ <argLine>-Xmx1024m</argLine>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <log.level>WARN</log.level>
+ </systemPropertyVariables>
+ <forkMode>always</forkMode>
+ <threadCount>1</threadCount>
+ <perCoreThreadCount>false</perCoreThreadCount>
+ </configuration>
+ </plugin>
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>
+ org.apache.maven.plugins
+ </groupId>
+ <artifactId>
+ maven-assembly-plugin
+ </artifactId>
+ <versionRange>
+ [2.4,)
+ </versionRange>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
new file mode 100644
index 0000000..c417249
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CoGroupITCase.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class CoGroupITCase implements Serializable {
+
+ @Test
+ public void testCoGroupLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, String>> left = env.fromElements(
+ new Tuple2<Integer, String>(1, "hello"),
+ new Tuple2<Integer, String>(2, "what's"),
+ new Tuple2<Integer, String>(2, "up")
+ );
+ DataSet<Tuple2<Integer, String>> right = env.fromElements(
+ new Tuple2<Integer, String>(1, "not"),
+ new Tuple2<Integer, String>(1, "much"),
+ new Tuple2<Integer, String>(2, "really")
+ );
+ DataSet<Tuple2<Integer,String>> joined = left.coGroup(right).where(0).equalTo(0)
+ .with((values1, values2, out) -> {
+ int sum = 0;
+ String conc = "";
+ while (values1.hasNext()) {
+ sum += values1.next().f0;
+ conc += values1.next().f1;
+ }
+ while (values2.hasNext()) {
+ sum += values2.next().f0;
+ conc += values2.next().f1;
+ }
+ });
+ env.execute();
+
+
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
new file mode 100644
index 0000000..f8d217e
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/CrossITCase.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class CrossITCase implements Serializable {
+
+ @Test
+ public void testCrossLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, String>> left = env.fromElements(
+ new Tuple2<Integer, String>(1, "hello"),
+ new Tuple2<Integer, String>(2, "what's"),
+ new Tuple2<Integer, String>(2, "up")
+ );
+ DataSet<Tuple2<Integer, String>> right = env.fromElements(
+ new Tuple2<Integer, String>(1, "not"),
+ new Tuple2<Integer, String>(1, "much"),
+ new Tuple2<Integer, String>(2, "really")
+ );
+ DataSet<Tuple2<Integer,String>> joined = left.cross(right)
+ .with((t,s) -> new Tuple2<Integer, String> (t.f0 + s.f0, t.f1 + " " + s.f1));
+
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
new file mode 100644
index 0000000..c775425
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FilterITCase.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.test.util.JavaProgramTestBase;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class FilterITCase extends JavaProgramTestBase {
+
+
+ public static DataSet<Tuple3<Integer, Long, String>> get3TupleDataSet(ExecutionEnvironment env) {
+
+ List<Tuple3<Integer, Long, String>> data = new ArrayList<Tuple3<Integer, Long, String>>();
+ data.add(new Tuple3<Integer, Long, String>(1,1l,"Hi"));
+ data.add(new Tuple3<Integer, Long, String>(2,2l,"Hello"));
+ data.add(new Tuple3<Integer, Long, String>(3,2l,"Hello world"));
+ data.add(new Tuple3<Integer, Long, String>(4,3l,"Hello world, how are you?"));
+ data.add(new Tuple3<Integer, Long, String>(5,3l,"I am fine."));
+ data.add(new Tuple3<Integer, Long, String>(6,3l,"Luke Skywalker"));
+ data.add(new Tuple3<Integer, Long, String>(7,4l,"Comment#1"));
+ data.add(new Tuple3<Integer, Long, String>(8,4l,"Comment#2"));
+ data.add(new Tuple3<Integer, Long, String>(9,4l,"Comment#3"));
+ data.add(new Tuple3<Integer, Long, String>(10,4l,"Comment#4"));
+ data.add(new Tuple3<Integer, Long, String>(11,5l,"Comment#5"));
+ data.add(new Tuple3<Integer, Long, String>(12,5l,"Comment#6"));
+ data.add(new Tuple3<Integer, Long, String>(13,5l,"Comment#7"));
+ data.add(new Tuple3<Integer, Long, String>(14,5l,"Comment#8"));
+ data.add(new Tuple3<Integer, Long, String>(15,5l,"Comment#9"));
+ data.add(new Tuple3<Integer, Long, String>(16,6l,"Comment#10"));
+ data.add(new Tuple3<Integer, Long, String>(17,6l,"Comment#11"));
+ data.add(new Tuple3<Integer, Long, String>(18,6l,"Comment#12"));
+ data.add(new Tuple3<Integer, Long, String>(19,6l,"Comment#13"));
+ data.add(new Tuple3<Integer, Long, String>(20,6l,"Comment#14"));
+ data.add(new Tuple3<Integer, Long, String>(21,6l,"Comment#15"));
+
+ Collections.shuffle(data);
+
+ return env.fromCollection(data);
+ }
+
+ private static int NUM_PROGRAMS = 1;
+
+ private int curProgId = config.getInteger("ProgramId", -1);
+ private String resultPath;
+ private String expectedResult;
+
+ public FilterITCase(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ expectedResult = FilterProgs.runProgram(curProgId, resultPath);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(expectedResult, resultPath);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
+
+ LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
+
+ for(int i=1; i <= NUM_PROGRAMS; i++) {
+ Configuration config = new Configuration();
+ config.setInteger("ProgramId", i);
+ tConfigs.add(config);
+ }
+
+ return toParameterList(tConfigs);
+ }
+
+ private static class FilterProgs {
+
+ public static String runProgram(int progId, String resultPath) throws Exception {
+
+ switch(progId) {
+ case 1: {
+ /*
+ * Test lambda filter
+ * Functionality identical to org.apache.flink.test.javaApiOperators.FilterITCase test 3
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds = get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
+ filter(value -> value.f2.contains("world"));
+ filterDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "3,2,Hello world\n" +
+ "4,3,Hello world, how are you?\n";
+ }
+ default:
+ throw new IllegalArgumentException("Invalid program id");
+ }
+
+ }
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
new file mode 100644
index 0000000..043b4e8
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatJoinITCase.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class FlatJoinITCase implements Serializable {
+
+ @Test
+ public void testFlatJoinLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, String>> left = env.fromElements(
+ new Tuple2<Integer, String>(1, "hello"),
+ new Tuple2<Integer, String>(2, "what's"),
+ new Tuple2<Integer, String>(2, "up")
+ );
+ DataSet<Tuple2<Integer, String>> right = env.fromElements(
+ new Tuple2<Integer, String>(1, "not"),
+ new Tuple2<Integer, String>(1, "much"),
+ new Tuple2<Integer, String>(2, "really")
+ );
+ DataSet<Tuple2<Integer,String>> joined = left.join(right).where(0).equalTo(0)
+ .with((t,s,out) -> out.collect(new Tuple2<Integer,String>(t.f0, t.f1 + " " + s.f1)));
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
new file mode 100644
index 0000000..55f507c
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/FlatMapITCase.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class FlatMapITCase implements Serializable {
+
+ @Test
+ public void testFlatMapLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> stringDs = env.fromElements("aa", "ab", "ac", "ad");
+ DataSet<String> flatMappedDs = stringDs.flatMap((s, out) -> out.collect(s.replace("a", "b")));
+ env.execute();
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
new file mode 100644
index 0000000..494aff6
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/GroupReduceITCase.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class GroupReduceITCase implements Serializable {
+
+ @Test
+ public void testAllGroupReduceLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> stringDs = env.fromElements("aa", "ab", "ac", "ad");
+ DataSet<String> concatDs = stringDs.reduceGroup((values, out) -> {
+ String conc = "";
+ while (values.hasNext()) {
+ String s = values.next();
+ conc = conc.concat(s);
+ }
+ out.collect(conc);
+ });
+ env.execute();
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testGroupReduceLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer,String>> stringDs = env.fromElements(
+ new Tuple2<Integer,String>(1, "aa"),
+ new Tuple2<Integer,String>(2, "ab"),
+ new Tuple2<Integer,String>(1, "ac"),
+ new Tuple2<Integer,String>(2, "ad")
+ );
+ DataSet<String> concatDs = stringDs
+ .groupBy(0)
+ .reduceGroup((values, out) -> {
+ String conc = "";
+ while (values.hasNext()) {
+ String s = values.next().f1;
+ conc = conc.concat(s);
+ }
+ out.collect(conc);
+ });
+ env.execute();
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
new file mode 100644
index 0000000..3f4f696
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/JoinITCase.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class JoinITCase implements Serializable {
+
+ @Test
+ public void testJoinLambda() {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Integer, String>> left = env.fromElements(
+ new Tuple2<Integer, String>(1, "hello"),
+ new Tuple2<Integer, String>(2, "what's"),
+ new Tuple2<Integer, String>(2, "up")
+ );
+ DataSet<Tuple2<Integer, String>> right = env.fromElements(
+ new Tuple2<Integer, String>(1, "not"),
+ new Tuple2<Integer, String>(1, "much"),
+ new Tuple2<Integer, String>(2, "really")
+ );
+ DataSet<Tuple2<Integer,String>> joined = left.join(right).where(0).equalTo(0)
+ .with((t,s) -> new Tuple2<Integer,String>(t.f0, t.f1 + " " + t.f1));
+
+ } catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
new file mode 100644
index 0000000..3af360b
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/MapITCase.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.UnsupportedLambdaExpressionException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class MapITCase implements Serializable{
+
+ @Test
+ public void TestMapLambda () {
+ try {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> stringDs = env.fromElements("aa", "ab", "ac", "ad");
+ DataSet<String> mappedDs = stringDs.map (s -> s.replace("a", "b"));
+ env.execute();
+ }
+ catch (UnsupportedLambdaExpressionException e) {
+ // Success
+ return;
+ }
+ catch (Exception e) {
+ Assert.fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
new file mode 100644
index 0000000..ab27fe4
--- /dev/null
+++ b/flink-java8-tests/src/test/java/org/apache/flink/test/javaApiOperators/lambdas/ReduceITCase.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.javaApiOperators.lambdas;
+
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.typeutils.BasicTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.test.util.JavaProgramTestBase;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class ReduceITCase extends JavaProgramTestBase {
+
+ public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> get5TupleDataSet(ExecutionEnvironment env) {
+
+ List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<Tuple5<Integer, Long, Integer, String, Long>>();
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(1,1l,0,"Hallo",1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,2l,1,"Hallo Welt",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,3l,2,"Hallo Welt wie",1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,4l,3,"Hallo Welt wie gehts?",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,5l,4,"ABC",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,6l,5,"BCD",3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,7l,6,"CDE",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,8l,7,"DEF",1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,9l,8,"EFG",1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,10l,9,"FGH",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,11l,10,"GHI",1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,12l,11,"HIJ",3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,13l,12,"IJK",3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,14l,13,"JKL",2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,15l,14,"KLM",2l));
+
+ Collections.shuffle(data);
+
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO
+ );
+
+ return env.fromCollection(data, type);
+ }
+
+ private static int NUM_PROGRAMS = 1;
+
+ private int curProgId = config.getInteger("ProgramId", -1);
+ private String resultPath;
+ private String expectedResult;
+
+ public ReduceITCase(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ protected void preSubmit() throws Exception {
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ expectedResult = ReduceProgs.runProgram(curProgId, resultPath);
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(expectedResult, resultPath);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {
+
+ LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();
+
+ for(int i=1; i <= NUM_PROGRAMS; i++) {
+ Configuration config = new Configuration();
+ config.setInteger("ProgramId", i);
+ tConfigs.add(config);
+ }
+
+ return toParameterList(tConfigs);
+ }
+
+ private static class ReduceProgs {
+
+ public static String runProgram(int progId, String resultPath) throws Exception {
+
+ switch(progId) {
+ case 1: {
+ /*
+ * Test reduce with lambda
+ * Functionality identical to org.apache.flink.test.javaApiOperators.ReduceITCase test 2
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = get5TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds
+ .groupBy(4, 0)
+ .reduce((in1, in2) -> {
+ Tuple5<Integer, Long, Integer, String, Long> out = new Tuple5<Integer, Long, Integer, String, Long>();
+ out.setFields(in1.f0, in1.f1 + in2.f1, 0, "P-)", in1.f4);
+ return out;
+ });
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,0,Hallo,1\n" +
+ "2,3,2,Hallo Welt wie,1\n" +
+ "2,2,1,Hallo Welt,2\n" +
+ "3,9,0,P-),2\n" +
+ "3,6,5,BCD,3\n" +
+ "4,17,0,P-),1\n" +
+ "4,17,0,P-),2\n" +
+ "5,11,10,GHI,1\n" +
+ "5,29,0,P-),2\n" +
+ "5,25,0,P-),3\n";
+ }
+ default:
+ throw new IllegalArgumentException("Invalid program id");
+ }
+
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 636c492..34cd232 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -23,7 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.common.aggregators.LongSumAggregator;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.IterationRuntimeContext;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -54,7 +54,7 @@ import java.io.IOException;
/**
* The base class for all tasks able to participate in an iteration.
*/
-public abstract class AbstractIterativePactTask<S extends Function, OT> extends RegularPactTask<S, OT>
+public abstract class AbstractIterativePactTask<S extends RichFunction, OT> extends RegularPactTask<S, OT>
implements Terminable
{
private static final Log log = LogFactory.getLog(AbstractIterativePactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index d7f3b50..7a77cff 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -75,7 +75,7 @@ import org.apache.flink.util.MutableObjectIterator;
* The type of the feed-back data set (bulk partial solution / workset). For bulk iterations, {@code Y} is the
* same as {@code X}
*/
-public class IterationHeadPactTask<X, Y, S extends Function, OT> extends AbstractIterativePactTask<S, OT> {
+public class IterationHeadPactTask<X, Y, S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT> {
private static final Log log = LogFactory.getLog(IterationHeadPactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index 25a6149..2a8325c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.runtime.io.network.api.BufferWriter;
import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
@@ -41,7 +41,7 @@ import org.apache.flink.util.Collector;
* a {@link BlockingBackChannel} for the workset -XOR- a {@link MutableHashTable} for the solution set. In this case
* this task must be scheduled on the same instance as the head.
*/
-public class IterationIntermediatePactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT> {
+public class IterationIntermediatePactTask<S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT> {
private static final Log log = LogFactory.getLog(IterationIntermediatePactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index 570630f..942e2f6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.iterative.task;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrier;
import org.apache.flink.runtime.iterative.concurrent.SolutionSetUpdateBarrierBroker;
import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
@@ -38,7 +38,7 @@ import org.apache.flink.util.Collector;
* <p/>
* If there is a separate solution set tail, the iteration head has to make sure to wait for it to finish.
*/
-public class IterationTailPactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT>
+public class IterationTailPactTask<S extends RichFunction, OT> extends AbstractIterativePactTask<S, OT>
implements PactTaskContext<S, OT> {
private static final Log log = LogFactory.getLog(IterationTailPactTask.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/22b24f20/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AbstractCachedBuildSideMatchDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AbstractCachedBuildSideMatchDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AbstractCachedBuildSideMatchDriver.java
index fe70171..d7d63af 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AbstractCachedBuildSideMatchDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/AbstractCachedBuildSideMatchDriver.java
@@ -19,7 +19,7 @@
package org.apache.flink.runtime.operators;
-import org.apache.flink.api.common.functions.GenericJoiner;
+import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -30,7 +30,7 @@ import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
-public abstract class AbstractCachedBuildSideMatchDriver<IT1, IT2, OT> extends MatchDriver<IT1, IT2, OT> implements ResettablePactDriver<GenericJoiner<IT1, IT2, OT>, OT> {
+public abstract class AbstractCachedBuildSideMatchDriver<IT1, IT2, OT> extends MatchDriver<IT1, IT2, OT> implements ResettablePactDriver<FlatJoinFunction<IT1, IT2, OT>, OT> {
private volatile JoinTaskIterator<IT1, IT2, OT> matchIterator;
@@ -110,7 +110,7 @@ public abstract class AbstractCachedBuildSideMatchDriver<IT1, IT2, OT> extends M
@Override
public void run() throws Exception {
- final GenericJoiner<IT1, IT2, OT> matchStub = this.taskContext.getStub();
+ final FlatJoinFunction<IT1, IT2, OT> matchStub = this.taskContext.getStub();
final Collector<OT> collector = this.taskContext.getOutputCollector();
if (buildSideIndex == 0) {