You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2014/10/18 17:17:16 UTC
[1/9] git commit: Prevent RPC service from blocking on exceptions
during task cleanup on canceling.
Repository: incubator-flink
Updated Branches:
refs/heads/master 23e30f09d -> a0ad90314
Prevent RPC service from blocking on exceptions during task cleanup on canceling.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/ed1827c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/ed1827c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/ed1827c7
Branch: refs/heads/master
Commit: ed1827c7ba159b16d6cfbb436217f3e0aea6155a
Parents: 23e30f0
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Oct 14 21:08:28 2014 +0200
Committer: Robert Metzger <me...@web.de>
Committed: Sat Oct 18 17:16:16 2014 +0200
----------------------------------------------------------------------
.../runtime/io/network/ChannelManager.java | 6 ++---
.../flink/runtime/taskmanager/TaskManager.java | 26 +++++++++++++-------
2 files changed, 19 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ed1827c7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ChannelManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ChannelManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ChannelManager.java
index 8f91814..e48f3af 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ChannelManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/ChannelManager.java
@@ -206,9 +206,8 @@ public class ChannelManager implements EnvelopeDispatcher, BufferProviderBroker
Channel channel = this.channels.remove(id);
if (channel != null) {
channel.destroy();
+ this.receiverCache.remove(channel);
}
-
- this.receiverCache.remove(channel);
}
// destroy and remove INPUT channels from registered channels and cache
@@ -216,9 +215,8 @@ public class ChannelManager implements EnvelopeDispatcher, BufferProviderBroker
Channel channel = this.channels.remove(id);
if (channel != null) {
channel.destroy();
+ this.receiverCache.remove(channel);
}
-
- this.receiverCache.remove(channel);
}
// clear and remove INPUT side buffer pools
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/ed1827c7/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
index f27f311..5c1cdab 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java
@@ -61,6 +61,7 @@ import org.apache.flink.core.protocols.VersionedProtocol;
import org.apache.flink.runtime.ExecutionMode;
import org.apache.flink.runtime.blob.BlobCache;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.RuntimeEnvironment;
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager;
@@ -572,7 +573,6 @@ public class TaskManager implements TaskOperationProtocol {
Task task = null;
boolean jarsRegistered = false;
- boolean success = false;
try {
// Now register data with the library manager
@@ -616,7 +616,7 @@ public class TaskManager implements TaskOperationProtocol {
env.addCopyTasksForCacheFile(cpTasks);
if (!task.startExecution()) {
- return new TaskOperationResult(executionId, false, "Task was canceled or failed.");
+ throw new CancelTaskException();
}
// final check that we can go (we do this after the registration, so the the "happen's before"
@@ -624,16 +624,19 @@ public class TaskManager implements TaskOperationProtocol {
if (shutdownStarted.get()) {
throw new Exception("Task Manager is shut down.");
}
-
- success = true;
+
return new TaskOperationResult(executionId, true);
}
catch (Throwable t) {
- LOG.error("Could not instantiate task", t);
- return new TaskOperationResult(executionId, false, ExceptionUtils.stringifyException(t));
- }
- finally {
- if (!success) {
+ String message;
+ if (t instanceof CancelTaskException) {
+ message = "Task was canceled";
+ } else {
+ LOG.error("Could not instantiate task", t);
+ message = ExceptionUtils.stringifyException(t);
+ }
+
+ try {
this.runningTasks.remove(executionId);
if (task != null) {
@@ -643,6 +646,11 @@ public class TaskManager implements TaskOperationProtocol {
libraryCacheManager.unregister(jobID);
}
}
+ catch (Throwable t2) {
+ LOG.error("Error during cleanup of task deployment", t2);
+ }
+
+ return new TaskOperationResult(executionId, false, message);
}
}
[7/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
deleted file mode 100644
index fe1dd43..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-import org.junit.Assert
-import org.apache.flink.api.common.InvalidProgramException
-import org.apache.flink.api.common.operators.Order
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-
-class GroupingTest {
-
- private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
- private val customTypeData = Array[CustomType](new CustomType())
- private val emptyLongData = Array[Long]()
-
- @Test
- def testGroupByKeyIndices1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- tupleDs.groupBy(0)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testGroupByKeyIndices2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work, grouping on basic type
- longDs.groupBy(0)
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testGroupByKeyIndices3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work, field position key on custom type
- customDs.groupBy(0)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyIndices4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, fiels position out of range
- tupleDs.groupBy(5)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyIndices5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, negative field position
- tupleDs.groupBy(-1)
- }
-
- @Test
- def testGroupByKeyFields1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- tupleDs.groupBy("_1")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyFields2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work, grouping on basic type
- longDs.groupBy("_1")
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyFields3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work, field key on custom type
- customDs.groupBy("_1")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testGroupByKeyFields4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, invalid field
- tupleDs.groupBy("foo")
- }
-
- @Test
- def testGroupByKeyFields5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work
- customDs.groupBy("myInt")
- }
-
- @Test
- def testGroupByKeyExpressions1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(customTypeData)
-
- // should work
- try {
- ds.groupBy("myInt")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyExpressions2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- // should not work: groups on basic type
- val longDs = env.fromCollection(emptyLongData)
- longDs.groupBy("l")
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testGroupByKeyExpressions3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work: groups on custom type
- customDs.groupBy(0)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupByKeyExpressions4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(customTypeData)
-
- // should not work, non-existent field
- ds.groupBy("myNonExistent")
- }
-
- @Test
- def testGroupByKeySelector1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- try {
- val customDs = env.fromCollection(customTypeData)
- customDs.groupBy { _.myLong }
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testGroupSortKeyFields1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
- try {
- tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testGroupSortKeyFields2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, field position out of range
- tupleDs.groupBy(0).sortGroup(5, Order.ASCENDING)
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testGroupSortKeyFields3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val longDs = env.fromCollection(emptyLongData)
- longDs.groupBy { x: Long => x } .sortGroup(0, Order.ASCENDING)
- }
-
- @Test
- def testChainedGroupSortKeyFields(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
- try {
- tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING).sortGroup(2, Order.DESCENDING)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
deleted file mode 100644
index 2605830..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichJoinFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.apache.flink.util.Collector
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object JoinProgs {
- var NUM_PROGRAMS: Int = 20
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * UDF Join on tuples with key field positions
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
-
- case 2 =>
- /*
- * UDF Join on tuples with multiple key field positions
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(0, 1).equalTo(0, 4) { (l, r) => (l._3, r._4) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt wie gehts?\n" +
- "Hello world,ABC\n" + "I am fine.,HIJ\n" + "I am fine.,IJK\n"
-
- case 3 =>
- /*
- * Default Join on tuples
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(0).equalTo(2)
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" + "(2,2,Hello),(2,3,2,Hallo Welt wie," +
- "1)\n" + "(3,2,Hello world),(3,4,3,Hallo Welt wie gehts?,2)\n"
-
- case 4 =>
- /*
- * Join with Huge
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.joinWithHuge(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
-
- case 5 =>
- /*
- * Join with Tiny
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.joinWithTiny(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
-
- case 6 =>
- /*
- * Join that returns the left input object
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => l }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n"
-
- case 7 =>
- /*
- * Join that returns the right input object
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => r }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt,2\n" + "2,2,1,Hallo Welt,2\n"
-
- case 8 =>
- /*
- * Join with broadcast set
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val intDs = CollectionDataSets.getIntDataSet(env)
- val ds1 = CollectionDataSets.get3TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where(1).equalTo(4).apply(
- new RichJoinFunction[
- (Int, Long, String),
- (Int, Long, Int, String, Long),
- (String, String, Int)] {
- private var broadcast = 41
-
- override def open(config: Configuration) {
- val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- broadcast = ints.sum
- }
-
- override def join(
- first: (Int, Long, String),
- second: (Int, Long, Int, String, Long)): (String, String, Int) = {
- (first._3, second. _4, broadcast)
- }
- }
- ).withBroadcastSet(intDs, "ints")
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo,55\n" + "Hi,Hallo Welt wie,55\n" + "Hello,Hallo Welt," +
- "55\n" + "Hello world,Hallo Welt,55\n"
-
- case 9 =>
- /*
- * Join on a tuple input with key field selector and a custom type input with key extractor
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallCustomTypeDataSet(env)
- val ds2 = CollectionDataSets.get3TupleDataSet(env)
- val joinDs = ds1.join(ds2).where( _.myInt ).equalTo(0) { (l, r) => (l.myString, r._3) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hi\n" + "Hello,Hello\n" + "Hello world,Hello\n"
-
- case 10 => // 12 in Java ITCase
- /*
- * Join on a tuple input with key field selector and a custom type input with key extractor
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
- val joinDs = ds1.join(ds2).where(1).equalTo(_.myLong) apply { (l, r) => (l._3, r.myString) }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hello\n" + "Hello,Hello world\n" + "Hello world,Hello world\n"
-
- case 11 => // 13 in Java ITCase
- /*
- * (Default) Join on two custom type inputs with key extractors
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getCustomTypeDataSet(env)
- val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
- val joinDs = ds1.join(ds2).where(_.myInt).equalTo(_.myInt)
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "1,0,Hi,1,0,Hi\n" + "2,1,Hello,2,1,Hello\n" + "2,1,Hello,2,2,Hello world\n" + "2," +
- "2,Hello world,2,1,Hello\n" + "2,2,Hello world,2,2,Hello world\n"
-
- case 12 => // 14 in Java ITCase
- /*
- * UDF Join on tuples with tuple-returning key selectors
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get3TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val joinDs = ds1.join(ds2).where( t => (t._1, t._2)).equalTo( t => (t._1, t._5)) apply {
- (l, r) => (l._3, r._4)
- }
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt wie gehts?\n" +
- "Hello world,ABC\n" + "I am fine.,HIJ\n" + "I am fine.,IJK\n"
-
- /**
- * Joins with POJOs
- */
- case 13 => // 15 in Java ITCase
- /*
- * Join nested pojo against tuple (selected using a string)
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val joinDs = ds1.join(ds2).where("nestedPojo.longNumber").equalTo("_7")
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
- "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
- "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
-
- case 14 => // 16 in Java ITCase
- /*
- * Join nested pojo against tuple (selected as an integer)
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val joinDs = ds1.join(ds2).where("nestedPojo.longNumber").equalTo(6) // <-- difference
- joinDs.writeAsCsv(resultPath)
- env.execute()
- "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
- "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
- "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
-
- case 15 => // 17 in Java ITCase
- /*
- * selecting multiple fields using expression language
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val joinDs = ds1.join(ds2)
- .where("nestedPojo.longNumber", "number", "str")
- .equalTo("_7", "_1", "_2")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
- "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
- "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
-
- case 16 => // 18 in Java ITCase
- /*
- * nested into tuple
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val joinDs = ds1.join(ds2).where("nestedPojo.longNumber", "number",
- "nestedTupleWithCustom._1").equalTo("_7", "_1", "_3")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
- "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
- "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
-
- case 17 => // 19 in Java ITCase
- /*
- * nested into tuple into pojo
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val joinDs = ds1.join(ds2)
- .where("nestedTupleWithCustom._1",
- "nestedTupleWithCustom._2.myInt",
- "nestedTupleWithCustom._2.myLong")
- .equalTo("_3", "_4", "_5")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
- "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
- "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
-
- case 18 => // 20 in Java ITCase
- /*
- * Non-POJO test to verify that full-tuple keys are working.
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env)
- val ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env)
- val joinDs = ds1.join(ds2).where(0).equalTo("_1._1", "_1._2")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "((1,1),one),((1,1),one)\n" + "((2,2),two),((2,2),two)\n" + "((3,3),three),((3,3)," +
- "three)\n"
-
- case 19 => // 21 in Java ITCase
- /*
- * Non-POJO test to verify "nested" tuple-element selection.
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env)
- val ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env)
- val joinDs = ds1.join(ds2).where("_1._1").equalTo("_1._1")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "((1,1),one),((1,1),one)\n" + "((2,2),two),((2,2),two)\n" + "((3,3),three),((3,3),three)\n"
-
- case 20 => // 22 in Java ITCase
- /*
- * full pojo with full tuple
- */
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedDataSetMatchingPojo(env)
- val joinDs = ds1.join(ds2).where("*").equalTo("*")
- joinDs.writeAsCsv(resultPath)
- env.setDegreeOfParallelism(1)
- env.execute()
- "1 First (10,100,1000,One) 10000,(10000,10,100,1000,One,1,First)\n" +
- "2 Second (20,200,2000,Two) 20000,(20000,20,200,2000,Two,2,Second)\n" +
- "3 Third (30,300,3000,Three) 30000,(30000,30,300,3000,Three,3,Third)\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id: " + progId)
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class JoinITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = JoinProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object JoinITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to JoinProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
deleted file mode 100644
index 0219154..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-import org.junit.Assert
-import org.apache.flink.api.common.InvalidProgramException
-import org.junit.Ignore
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class JoinOperatorTest {
-
- private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
- private val customTypeData = Array[CustomType](new CustomType())
- private val emptyLongData = Array[Long]()
-
- @Test
- def testJoinKeyIndices1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- ds1.join(ds2).where(0).equalTo(0)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyIndices2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, incompatible key types
- ds1.join(ds2).where(0).equalTo(2)
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyIndices3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, non-matching number of key indices
- ds1.join(ds2).where(0, 1).equalTo(2)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testJoinKeyIndices4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, index out of range
- ds1.join(ds2).where(5).equalTo(0)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testJoinKeyIndices5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, negative position
- ds1.join(ds2).where(-1).equalTo(-1)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testJoinKeyIndices6(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, key index on custom type
- ds1.join(ds2).where(5).equalTo(0)
- }
-
- @Test
- def testJoinKeyFields1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- ds1.join(ds2).where("_1").equalTo("_1")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyFields2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, incompatible field types
- ds1.join(ds2).where("_1").equalTo("_3")
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyFields3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, non-matching number of key indices
-
- ds1.join(ds2).where("_1", "_2").equalTo("_3")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testJoinKeyFields4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, non-existent key
- ds1.join(ds2).where("foo").equalTo("_1")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testJoinKeyFields5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should not work, non-matching number of key indices
- ds1.join(ds2).where("_1").equalTo("bar")
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testJoinKeyFields6(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, field key on custom type
- ds1.join(ds2).where("_2").equalTo("_1")
- }
-
- @Test
- def testJoinKeyExpressions1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should work
- try {
- ds1.join(ds2).where("myInt").equalTo("myInt")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyExpressions2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, incompatible join key types
- ds1.join(ds2).where("myInt").equalTo("myString")
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyExpressions3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, incompatible number of keys
- ds1.join(ds2).where("myInt", "myString").equalTo("myInt")
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testJoinKeyExpressions4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, join key non-existent
- ds1.join(ds2).where("myNonExistent").equalTo("i")
- }
-
- @Test
- def testJoinKeySelectors1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should work
- try {
- ds1.join(ds2).where { _.myLong} equalTo { _.myLong }
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testJoinKeyMixing1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- ds1.join(ds2).where { _.myLong }.equalTo(3)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testJoinKeyMixing2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should work
- try {
- ds1.join(ds2).where(3).equalTo { _.myLong }
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyMixing3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, incompatible types
- ds1.join(ds2).where(2).equalTo { _.myLong }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testJoinKeyMixing4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, more than one field position key
- ds1.join(ds2).where(1, 3) equalTo { _.myLong }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
deleted file mode 100644
index 71c6b8b..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichMapFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.apache.flink.util.Collector
-import org.junit.Assert
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object MapProgs {
- var NUM_PROGRAMS: Int = 9
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Test identity map with basic type
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getStringDataSet(env)
- val identityMapDs = ds.map( t => t)
- identityMapDs.writeAsText(resultPath)
- env.execute()
- "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
- ".\n" + "Luke Skywalker\n" + "Random comment\n" + "LOL\n"
-
- case 2 =>
- /*
- * Test identity map with a tuple
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val identityMapDs = ds.map( t => t )
- identityMapDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
- "how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" + "7,4," +
- "Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" + "11,5," +
- "Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
- "Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
- "6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
-
- case 3 =>
- /*
- * Test type conversion mapper (Custom -> Tuple)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val typeConversionMapDs = ds.map( c => (c.myInt, c.myLong, c.myString) )
- typeConversionMapDs.writeAsCsv(resultPath)
- env.execute()
- "1,0,Hi\n" + "2,1,Hello\n" + "2,2,Hello world\n" + "3,3,Hello world, " +
- "how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n" + "4,6," +
- "Comment#1\n" + "4,7,Comment#2\n" + "4,8,Comment#3\n" + "4,9,Comment#4\n" + "5,10," +
- "Comment#5\n" + "5,11,Comment#6\n" + "5,12,Comment#7\n" + "5,13,Comment#8\n" + "5,14," +
- "Comment#9\n" + "6,15,Comment#10\n" + "6,16,Comment#11\n" + "6,17,Comment#12\n" + "6," +
- "18,Comment#13\n" + "6,19,Comment#14\n" + "6,20,Comment#15\n"
-
- case 4 =>
- /*
- * Test type conversion mapper (Tuple -> Basic)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val typeConversionMapDs = ds.map(_._3)
- typeConversionMapDs.writeAsText(resultPath)
- env.execute()
- "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
- ".\n" + "Luke Skywalker\n" + "Comment#1\n" + "Comment#2\n" + "Comment#3\n" +
- "Comment#4\n" + "Comment#5\n" + "Comment#6\n" + "Comment#7\n" + "Comment#8\n" +
- "Comment#9\n" + "Comment#10\n" + "Comment#11\n" + "Comment#12\n" + "Comment#13\n" +
- "Comment#14\n" + "Comment#15\n"
-
- case 5 =>
- /*
- * Test mapper on tuple - Increment Integer field, reorder second and third fields
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val tupleMapDs = ds.map( t => (t._1 + 1, t._3, t._2) )
- tupleMapDs.writeAsCsv(resultPath)
- env.execute()
- "2,Hi,1\n" + "3,Hello,2\n" + "4,Hello world,2\n" + "5,Hello world, how are you?," +
- "3\n" + "6,I am fine.,3\n" + "7,Luke Skywalker,3\n" + "8,Comment#1,4\n" + "9,Comment#2," +
- "4\n" + "10,Comment#3,4\n" + "11,Comment#4,4\n" + "12,Comment#5,5\n" + "13,Comment#6," +
- "5\n" + "14,Comment#7,5\n" + "15,Comment#8,5\n" + "16,Comment#9,5\n" + "17,Comment#10," +
- "6\n" + "18,Comment#11,6\n" + "19,Comment#12,6\n" + "20,Comment#13,6\n" + "21," +
- "Comment#14,6\n" + "22,Comment#15,6\n"
-
- case 6 =>
- /*
- * Test mapper on Custom - lowercase myString
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val customMapDs = ds.map { c => c.myString = c.myString.toLowerCase; c }
- customMapDs.writeAsText(resultPath)
- env.execute()
- "1,0,hi\n" + "2,1,hello\n" + "2,2,hello world\n" + "3,3,hello world, " +
- "how are you?\n" + "3,4,i am fine.\n" + "3,5,luke skywalker\n" + "4,6," +
- "comment#1\n" + "4,7,comment#2\n" + "4,8,comment#3\n" + "4,9,comment#4\n" + "5,10," +
- "comment#5\n" + "5,11,comment#6\n" + "5,12,comment#7\n" + "5,13,comment#8\n" + "5,14," +
- "comment#9\n" + "6,15,comment#10\n" + "6,16,comment#11\n" + "6,17,comment#12\n" + "6," +
- "18,comment#13\n" + "6,19,comment#14\n" + "6,20,comment#15\n"
-
- case 7 =>
- /*
- * Test mapper if UDF returns input object - increment first field of a tuple
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).map {
- t => MutableTuple3(t._1, t._2, t._3)
- }
- val inputObjMapDs = ds.map { t => t._1 = t._1 + 1; t }
- inputObjMapDs.writeAsCsv(resultPath)
- env.execute()
- "2,1,Hi\n" + "3,2,Hello\n" + "4,2,Hello world\n" + "5,3,Hello world, " +
- "how are you?\n" + "6,3,I am fine.\n" + "7,3,Luke Skywalker\n" + "8,4," +
- "Comment#1\n" + "9,4,Comment#2\n" + "10,4,Comment#3\n" + "11,4,Comment#4\n" + "12,5," +
- "Comment#5\n" + "13,5,Comment#6\n" + "14,5,Comment#7\n" + "15,5,Comment#8\n" + "16,5," +
- "Comment#9\n" + "17,6,Comment#10\n" + "18,6,Comment#11\n" + "19,6,Comment#12\n" + "20," +
- "6,Comment#13\n" + "21,6,Comment#14\n" + "22,6,Comment#15\n"
-
- case 8 =>
- /*
- * Test map with broadcast set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ints = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val bcMapDs = ds.map(
- new RichMapFunction[(Int, Long, String), (Int, Long, String)] {
- var f2Replace = 0
- override def open(config: Configuration): Unit = {
- val ints = getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- f2Replace = ints.sum
- }
- override def map(in: (Int, Long, String)): (Int, Long, String) = {
- in.copy(_1 = f2Replace)
- }
- }).withBroadcastSet(ints, "ints")
- bcMapDs.writeAsCsv(resultPath)
- env.execute()
- "55,1,Hi\n" + "55,2,Hello\n" + "55,2,Hello world\n" + "55,3,Hello world, " +
- "how are you?\n" + "55,3,I am fine.\n" + "55,3,Luke Skywalker\n" + "55,4," +
- "Comment#1\n" + "55,4,Comment#2\n" + "55,4,Comment#3\n" + "55,4,Comment#4\n" + "55,5," +
- "Comment#5\n" + "55,5,Comment#6\n" + "55,5,Comment#7\n" + "55,5,Comment#8\n" + "55,5," +
- "Comment#9\n" + "55,6,Comment#10\n" + "55,6,Comment#11\n" + "55,6,Comment#12\n" + "55," +
- "6,Comment#13\n" + "55,6,Comment#14\n" + "55,6,Comment#15\n"
-
- case 9 =>
- /*
- * Test passing configuration object.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
- val conf = new Configuration
- val testKey = "testVariable"
- val testValue = 666
- conf.setInteger(testKey, testValue)
- val bcMapDs = ds.map(
- new RichMapFunction[(Int, Long, String), (Int, Long, String)] {
- override def open(config: Configuration): Unit = {
- val fromConfig = config.getInteger(testKey, -1)
- Assert.assertEquals(testValue, fromConfig)
- }
- override def map(in: (Int, Long, String)): (Int, Long, String) = {
- in
- }
- }).withParameters(conf)
- bcMapDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class MapITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = MapProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object MapITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to MapProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
deleted file mode 100644
index 07d4897..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.{RichFilterFunction, RichMapFunction}
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object PartitionProgs {
- var NUM_PROGRAMS: Int = 7
-
- def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
- progId match {
- case 1 =>
- /*
- * Test hash partition by tuple field
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val unique = ds.partitionByHash(1).mapPartition( _.map(_._2).toSet )
-
- unique.writeAsText(resultPath)
- env.execute()
-
- "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
-
- case 2 =>
- /*
- * Test hash partition by key selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val unique = ds.partitionByHash( _._2 ).mapPartition( _.map(_._2).toSet )
-
- unique.writeAsText(resultPath)
- env.execute()
- "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
-
- case 3 =>
- /*
- * Test forced rebalancing
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.generateSequence(1, 3000)
-
- val skewed = ds.filter(_ > 780)
- val rebalanced = skewed.rebalance()
-
- val countsInPartition = rebalanced.map( new RichMapFunction[Long, (Int, Long)] {
- def map(in: Long) = {
- (getRuntimeContext.getIndexOfThisSubtask, 1)
- }
- })
- .groupBy(0)
- .reduce { (v1, v2) => (v1._1, v1._2 + v2._2) }
- // round counts to mitigate runtime scheduling effects (lazy split assignment)
- .map { in => (in._1, in._2 / 10) }
-
- countsInPartition.writeAsText(resultPath)
- env.execute()
-
- val numPerPartition : Int = 2220 / env.getDegreeOfParallelism / 10
- var result = ""
- for (i <- 0 until env.getDegreeOfParallelism) {
- result += "(" + i + "," + numPerPartition + ")\n"
- }
- result
-
- case 4 =>
- // Verify that mapPartition operation after repartition picks up correct
- // DOP
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- env.setDegreeOfParallelism(1)
-
- val unique = ds.partitionByHash(1)
- .setParallelism(4)
- .mapPartition( _.map(_._2).toSet )
-
- unique.writeAsText(resultPath)
- env.execute()
-
- "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
-
- case 5 =>
- // Verify that map operation after repartition picks up correct
- // DOP
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- env.setDegreeOfParallelism(1)
-
- val count = ds.partitionByHash(0).setParallelism(4).map(
- new RichMapFunction[(Int, Long, String), Tuple1[Int]] {
- var first = true
- override def map(in: (Int, Long, String)): Tuple1[Int] = {
- // only output one value with count 1
- if (first) {
- first = false
- Tuple1(1)
- } else {
- Tuple1(0)
- }
- }
- }).sum(0)
-
- count.writeAsText(resultPath)
- env.execute()
-
- if (onCollection) "(1)\n" else "(4)\n"
-
- case 6 =>
- // Verify that filter operation after repartition picks up correct
- // DOP
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- env.setDegreeOfParallelism(1)
-
- val count = ds.partitionByHash(0).setParallelism(4).filter(
- new RichFilterFunction[(Int, Long, String)] {
- var first = true
- override def filter(in: (Int, Long, String)): Boolean = {
- // only output one value with count 1
- if (first) {
- first = false
- true
- } else {
- false
- }
- }
- })
- .map( _ => Tuple1(1)).sum(0)
-
- count.writeAsText(resultPath)
- env.execute()
-
- if (onCollection) "(1)\n" else "(4)\n"
-
- case 7 =>
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(3)
- val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
- val uniqLongs = ds
- .partitionByHash("nestedPojo.longNumber")
- .setParallelism(4)
- .mapPartition( _.map(_.nestedPojo.longNumber).toSet )
-
- uniqLongs.writeAsText(resultPath)
- env.execute()
- "10000\n" + "20000\n" + "30000\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class PartitionITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = PartitionProgs.runProgram(curProgId, resultPath, isCollectionExecution)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object PartitionITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to PartitionProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
deleted file mode 100644
index 867eb81..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichReduceFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object ReduceProgs {
- var NUM_PROGRAMS: Int = 10
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Reduce on tuples with key field selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1)
- .reduce { (in1, in2) => (in1._1 + in2._1, in1._2, "B-)") }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,B-)\n" + "15,3,B-)\n" + "34,4,B-)\n" + "65,5,B-)\n" + "111,6,B-)\n"
-
- case 2 =>
- /*
- * Reduce on tuples with multiple key field selectors
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val reduceDs = ds.groupBy(4, 0)
- .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
- "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
- "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
-
- case 3 =>
- /*
- * Reduce on tuples with key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(_._2)
- .reduce { (in1, in2) => (in1._1 + in2._1, in1._2, "B-)") }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,B-)\n" + "15,3,B-)\n" + "34,4,B-)\n" + "65,5,B-)\n" + "111,6,B-)\n"
-
- case 4 =>
- /*
- * Reduce on custom type with key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val reduceDs = ds.groupBy(_.myInt)
- .reduce { (in1, in2) =>
- in1.myLong += in2.myLong
- in1.myString = "Hello!"
- in1
- }
- reduceDs.writeAsText(resultPath)
- env.execute()
- "1,0,Hi\n" + "2,3,Hello!\n" + "3,12,Hello!\n" + "4,30,Hello!\n" + "5,60," +
- "Hello!\n" + "6,105,Hello!\n"
-
- case 5 =>
- /*
- * All-reduce for tuple
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs =
- ds.reduce { (in1, in2) => (in1._1 + in2._1, in1._2 + in2._2, "Hello World") }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "231,91,Hello World\n"
-
- case 6 =>
- /*
- * All-reduce for custom types
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val reduceDs = ds
- .reduce { (in1, in2) =>
- in1.myInt += in2.myInt
- in1.myLong += in2.myLong
- in1.myString = "Hello!"
- in1
- }
- reduceDs.writeAsText(resultPath)
- env.execute()
- "91,210,Hello!"
-
- case 7 =>
- /*
- * Reduce with broadcast set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val intDs = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1).reduce(
- new RichReduceFunction[(Int, Long, String)] {
- private var f2Replace = ""
-
- override def open(config: Configuration) {
- val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- f2Replace = ints.sum + ""
- }
-
- override def reduce(
- in1: (Int, Long, String),
- in2: (Int, Long, String)): (Int, Long, String) = {
- (in1._1 + in2._1, in1._2, f2Replace)
- }
- }).withBroadcastSet(intDs, "ints")
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,55\n" + "15,3,55\n" + "34,4,55\n" + "65,5,55\n" + "111,6,55\n"
-
- case 8 =>
- /*
- * Reduce with UDF that returns the second input object (check mutable object handling)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).map (t => MutableTuple3(t._1, t._2, t._3))
- val reduceDs = ds.groupBy(1).reduce(
- new RichReduceFunction[MutableTuple3[Int, Long, String]] {
- override def reduce(
- in1: MutableTuple3[Int, Long, String],
- in2: MutableTuple3[Int, Long, String]): MutableTuple3[Int, Long, String] = {
- in2._1 = in1._1 + in2._1
- in2._3 = "Hi again!"
- in2
- }
- })
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,Hi again!\n" + "15,3,Hi again!\n" + "34,4,Hi again!\n" + "65,5," +
- "Hi again!\n" + "111,6,Hi again!\n"
-
- case 9 =>
- /*
- * Reduce with a Tuple-returning KeySelector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val reduceDs = ds.groupBy(t => (t._1, t._5))
- .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
- "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
- "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
-
- case 10 =>
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val reduceDs = ds.groupBy("_5", "_1")
- .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
- "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
- "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
-
- case id =>
- throw new IllegalArgumentException(s"Invalid program id $id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class ReduceITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = ReduceProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object ReduceITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to ReduceProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
deleted file mode 100644
index 488520e..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.java.aggregation.Aggregations
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-
-/**
- * These tests are copied from [[AggregateITCase]] replacing calls to aggregate with calls to sum,
- * min, and max
- */
-object SumMinMaxProgs {
- var NUM_PROGRAMS: Int = 3
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- // Full aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(10)
- // val ds = CollectionDataSets.get3TupleDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .sum(0)
- .andMax(1)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map{ t => (t._1, t._2) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "231,6\n"
-
- case 2 =>
- // Grouped aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .groupBy(1)
- .sum(0)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map { t => (t._2, t._1) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n"
-
- case 3 =>
- // Nested aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .groupBy(1)
- .min(0)
- .min(0)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map { t => new Tuple1(t._1) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "1\n"
-
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class SumMinMaxITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = SumMinMaxProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object SumMinMaxITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to SumMinMaxProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
deleted file mode 100644
index bf29b58..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.Assert
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object UnionProgs {
- var NUM_PROGRAMS: Int = 3
-
- private final val FULL_TUPLE_3_STRING: String = "1,1,Hi\n" + "2,2,Hello\n" + "3,2," +
- "Hello world\n" + "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3," +
- "Luke Skywalker\n" + "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4," +
- "Comment#4\n" + "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5," +
- "Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6," +
- "Comment#12\n" + "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Union of 2 Same Data Sets
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env))
- unionDs.writeAsCsv(resultPath)
- env.execute()
- FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING
-
- case 2 =>
- /*
- * Union of 5 same Data Sets, with multiple unions
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val unionDs = ds
- .union(CollectionDataSets.get3TupleDataSet(env))
- .union(CollectionDataSets.get3TupleDataSet(env))
- .union(CollectionDataSets.get3TupleDataSet(env))
- .union(CollectionDataSets.get3TupleDataSet(env))
- unionDs.writeAsCsv(resultPath)
- env.execute()
- FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING +
- FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING
-
- case 3 =>
- /*
- * Test on union with empty dataset
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- // Don't know how to make an empty result in an other way than filtering it
- val empty = CollectionDataSets.get3TupleDataSet(env).filter( t => false )
- val unionDs = CollectionDataSets.get3TupleDataSet(env).union(empty)
- unionDs.writeAsCsv(resultPath)
- env.execute()
- FULL_TUPLE_3_STRING
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class UnionITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = UnionProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object UnionITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to UnionProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
deleted file mode 100644
index c4d7dc8..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators.translation
-
-import org.apache.flink.api.common.Plan
-import org.apache.flink.api.common.operators.GenericDataSourceBase
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase
-
-import org.apache.flink.api.java.aggregation.Aggregations
-import org.apache.flink.api.scala._
-import org.junit.Assert.{assertEquals, assertTrue, fail}
-import org.junit.Test
-
-class AggregateTranslationTest {
- @Test
- def translateAggregate(): Unit = {
- try {
- val DOP = 8
-
- val env = ExecutionEnvironment.createLocalEnvironment(DOP)
-
- val initialData = env.fromElements((3.141592, "foobar", 77L))
-
- initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).print()
-
- val p: Plan = env.createProgramPlan()
- val sink = p.getDataSinks.iterator.next
-
- val reducer= sink.getInput.asInstanceOf[GroupReduceOperatorBase[_, _, _]]
-
- assertEquals(1, reducer.getKeyColumns(0).length)
- assertEquals(0, reducer.getKeyColumns(0)(0))
- assertEquals(-1, reducer.getDegreeOfParallelism)
- assertTrue(reducer.isCombinable)
- assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Test caused an error: " + e.getMessage)
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
deleted file mode 100644
index 4a60154..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators.translation
-
-import org.apache.flink.api.common.functions.{RichCoGroupFunction, RichMapFunction,
-RichJoinFunction}
-import org.apache.flink.api.common.operators.GenericDataSinkBase
-import org.apache.flink.api.java.operators.translation.WrappingFunction
-import org.junit.Assert.assertArrayEquals
-import org.junit.Assert.assertEquals
-import org.junit.Assert.fail
-import org.apache.flink.api.common.{InvalidProgramException, Plan}
-import org.apache.flink.api.common.aggregators.LongSumAggregator
-import org.apache.flink.api.common.operators.base.DeltaIterationBase
-import org.apache.flink.api.common.operators.base.JoinOperatorBase
-import org.apache.flink.api.common.operators.base.MapOperatorBase
-import org.junit.Test
-
-import org.apache.flink.util.Collector
-
-import org.apache.flink.api.scala._
-
-class DeltaIterationTranslationTest {
-
- @Test
- def testCorrectTranslation(): Unit = {
- try {
- val JOB_NAME = "Test JobName"
- val ITERATION_NAME = "Test Name"
- val BEFORE_NEXT_WORKSET_MAP = "Some Mapper"
- val AGGREGATOR_NAME = "AggregatorName"
- val ITERATION_KEYS = Array(2)
- val NUM_ITERATIONS = 13
- val DEFAULT_DOP = 133
- val ITERATION_DOP = 77
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(DEFAULT_DOP)
-
- val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
- val initialWorkSet = env.fromElements((1.23, "abc"))
-
- val result = initialSolutionSet.iterateDelta(initialWorkSet, NUM_ITERATIONS, ITERATION_KEYS) {
- (s, ws) =>
- val wsSelfJoin = ws.map(new IdentityMapper[(Double, String)]())
- .join(ws).where(1).equalTo(1) { (l, r) => l }
-
- val joined = wsSelfJoin.join(s).where(1).equalTo(2).apply(new SolutionWorksetJoin)
- (joined, joined.map(new NextWorksetMapper).name(BEFORE_NEXT_WORKSET_MAP))
- }
- result.name(ITERATION_NAME)
- .setParallelism(ITERATION_DOP)
- .registerAggregator(AGGREGATOR_NAME, new LongSumAggregator)
-
- result.print()
- result.writeAsText("/dev/null")
-
- val p: Plan = env.createProgramPlan(JOB_NAME)
- assertEquals(JOB_NAME, p.getJobName)
- assertEquals(DEFAULT_DOP, p.getDefaultParallelism)
- var sink1: GenericDataSinkBase[_] = null
- var sink2: GenericDataSinkBase[_] = null
- val sinks = p.getDataSinks.iterator
- sink1 = sinks.next
- sink2 = sinks.next
-
- val iteration: DeltaIterationBase[_, _] =
- sink1.getInput.asInstanceOf[DeltaIterationBase[_,_]]
-
- assertEquals(iteration, sink2.getInput)
- assertEquals(NUM_ITERATIONS, iteration.getMaximumNumberOfIterations)
- assertArrayEquals(ITERATION_KEYS, iteration.getSolutionSetKeyFields)
- assertEquals(ITERATION_DOP, iteration.getDegreeOfParallelism)
- assertEquals(ITERATION_NAME, iteration.getName)
-
- val nextWorksetMapper: MapOperatorBase[_, _, _] =
- iteration.getNextWorkset.asInstanceOf[MapOperatorBase[_, _, _]]
- val solutionSetJoin: JoinOperatorBase[_, _, _, _] =
- iteration.getSolutionSetDelta.asInstanceOf[JoinOperatorBase[_, _, _, _]]
- val worksetSelfJoin: JoinOperatorBase[_, _, _, _] =
- solutionSetJoin.getFirstInput.asInstanceOf[JoinOperatorBase[_, _, _, _]]
- val worksetMapper: MapOperatorBase[_, _, _] =
- worksetSelfJoin.getFirstInput.asInstanceOf[MapOperatorBase[_, _, _]]
-
- assertEquals(classOf[IdentityMapper[_]], worksetMapper.getUserCodeWrapper.getUserCodeClass)
-
-
- assertEquals(
- classOf[NextWorksetMapper],
- nextWorksetMapper.getUserCodeWrapper.getUserCodeClass)
-
-
- if (solutionSetJoin.getUserCodeWrapper.getUserCodeObject.isInstanceOf[WrappingFunction[_]]) {
- val wf: WrappingFunction[_] = solutionSetJoin.getUserCodeWrapper.getUserCodeObject
- .asInstanceOf[WrappingFunction[_]]
- assertEquals(classOf[SolutionWorksetJoin],
- wf.getWrappedFunction.getClass)
- }
- else {
- assertEquals(classOf[SolutionWorksetJoin],
- solutionSetJoin.getUserCodeWrapper.getUserCodeClass)
- }
-
- assertEquals(BEFORE_NEXT_WORKSET_MAP, nextWorksetMapper.getName)
- assertEquals(AGGREGATOR_NAME, iteration.getAggregators.getAllRegisteredAggregators.iterator
- .next.getName)
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail(e.getMessage)
- }
- }
- }
-
- @Test
- def testRejectWhenSolutionSetKeysDontMatchJoin(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
- val initialWorkSet = env.fromElements((1.23, "abc"))
-
- val iteration = initialSolutionSet.iterateDelta(initialWorkSet, 10, Array(0)) {
- (s, ws) =>
- try {
- ws.join(s).where(1).equalTo(2)
- fail("Accepted invalid program.")
- } catch {
- case e: InvalidProgramException => // all good
- }
- try {
- s.join(ws).where(2).equalTo(1)
- fail("Accepted invalid program.")
- } catch {
- case e: InvalidProgramException => // all good
- }
- (s, ws)
- }
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail(e.getMessage)
- }
- }
- }
-
- @Test
- def testRejectWhenSolutionSetKeysDontMatchCoGroup(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
- val initialWorkSet = env.fromElements((1.23, "abc"))
-
- val iteration = initialSolutionSet.iterateDelta(initialWorkSet, 10, Array(0)) {
- (s, ws) =>
- try {
- ws.coGroup(s).where(1).equalTo(2)
- fail("Accepted invalid program.")
- } catch {
- case e: InvalidProgramException => // all good
- }
- try {
- s.coGroup(ws).where(2).equalTo(1)
- fail("Accepted invalid program.")
- } catch {
- case e: InvalidProgramException => // all good
- }
- (s, ws)
- }
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail(e.getMessage)
- }
- }
- }
-//
-// @Test def testRejectWhenSolutionSetKeysDontMatchCoGroup {
-// try {
-// val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
-// @SuppressWarnings(Array("unchecked")) val initialSolutionSet: DataSet[Tuple3[Double, Long,
-// String]] = env.fromElements(new Tuple3[Double, Long, String](3.44, 5L, "abc"))
-// @SuppressWarnings(Array("unchecked")) val initialWorkSet: DataSet[Tuple2[Double,
-// String]] = env.fromElements(new Tuple2[Double, String](1.23, "abc"))
-// val iteration: DeltaIteration[Tuple3[Double, Long, String], Tuple2[Double,
-// String]] = initialSolutionSet.iterateDelta(initialWorkSet, 10, 1)
-// try {
-// iteration.getWorkset.coGroup(iteration.getSolutionSet).where(1).equalTo(2).`with`(
-// new DeltaIterationTranslationTest.SolutionWorksetCoGroup1)
-// fail("Accepted invalid program.")
-// }
-// catch {
-// case e: InvalidProgramException => {
-// }
-// }
-// try {
-// iteration.getSolutionSet.coGroup(iteration.getWorkset).where(2).equalTo(1).`with`(
-// new DeltaIterationTranslationTest.SolutionWorksetCoGroup2)
-// fail("Accepted invalid program.")
-// }
-// catch {
-// case e: InvalidProgramException => {
-// }
-// }
-// }
-// catch {
-// case e: Exception => {
-// System.err.println(e.getMessage)
-// e.printStackTrace
-// fail(e.getMessage)
-// }
-// }
-// }
-}
-
-class SolutionWorksetJoin
- extends RichJoinFunction[(Double, String), (Double, Long, String), (Double, Long, String)] {
- def join(first: (Double, String), second: (Double, Long, String)): (Double, Long, String) = {
- null
- }
-}
-
-class NextWorksetMapper extends RichMapFunction[(Double, Long, String), (Double, String)] {
- def map(value: (Double, Long, String)): (Double, String) = {
- null
- }
-}
-
-class IdentityMapper[T] extends RichMapFunction[T, T] {
- def map(value: T): T = {
- value
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
deleted file mode 100644
index 931d34d..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators.translation
-
-import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase
-import org.junit.Assert
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class DistinctTranslationTest {
- @Test
- def testCombinable(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val input = env.fromElements("1", "2", "1", "3")
-
- val op = input.distinct { x => x}
- op.print()
-
- val p = env.createProgramPlan()
-
- val reduceOp =
- p.getDataSinks.iterator.next.getInput.asInstanceOf[GroupReduceOperatorBase[_, _, _]]
-
- Assert.assertTrue(reduceOp.isCombinable)
- }
- catch {
- case e: Exception => {
- e.printStackTrace()
- Assert.fail(e.getMessage)
- }
- }
- }
-}
-
[5/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
new file mode 100644
index 0000000..1aad4d1
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala
+
+import java.lang.reflect.Method
+
+import org.apache.flink.api.java.operators._
+import org.apache.flink.api.java.{DataSet => JavaDataSet}
+
+import scala.language.existentials
+
+import org.junit.Assert._
+import org.junit.Test
+
+/**
+ * This checks whether the Scala API is up to feature parity with the Java API. Right now is very
+ * simple, it is only checked whether a method with the same name exists.
+ *
+ * When adding excluded methods to the lists you should give a good reason in a comment.
+ *
+ * Note: This is inspired by the JavaAPICompletenessChecker from Spark.
+ */
+class ScalaAPICompletenessTest {
+
+ private def isExcludedByName(method: Method): Boolean = {
+ val name = method.getDeclaringClass.getName + "." + method.getName
+ val excludedNames = Seq(
+ // These are only used internally. Should be internal API but Java doesn't have
+ // private[flink].
+ "org.apache.flink.api.java.DataSet.getExecutionEnvironment",
+ "org.apache.flink.api.java.DataSet.getType",
+ "org.apache.flink.api.java.operators.Operator.getResultType",
+ "org.apache.flink.api.java.operators.Operator.getName",
+ "org.apache.flink.api.java.operators.Grouping.getDataSet",
+ "org.apache.flink.api.java.operators.Grouping.getKeys",
+ "org.apache.flink.api.java.operators.SingleInputOperator.getInput",
+ "org.apache.flink.api.java.operators.SingleInputOperator.getInputType",
+ "org.apache.flink.api.java.operators.TwoInputOperator.getInput1",
+ "org.apache.flink.api.java.operators.TwoInputOperator.getInput2",
+ "org.apache.flink.api.java.operators.TwoInputOperator.getInput1Type",
+ "org.apache.flink.api.java.operators.TwoInputOperator.getInput2Type",
+ "org.apache.flink.api.java.ExecutionEnvironment.localExecutionIsAllowed",
+ "org.apache.flink.api.java.ExecutionEnvironment.setDefaultLocalParallelism",
+
+ // This is really just a mapper, which in Scala can easily expressed as a map lambda
+ "org.apache.flink.api.java.DataSet.writeAsFormattedText",
+
+ // Exclude minBy and maxBy for now, since there is some discussion about our aggregator
+ // semantics
+ "org.apache.flink.api.java.DataSet.minBy",
+ "org.apache.flink.api.java.DataSet.maxBy",
+ "org.apache.flink.api.java.operators.UnsortedGrouping.minBy",
+ "org.apache.flink.api.java.operators.UnsortedGrouping.maxBy"
+
+ )
+ val excludedPatterns = Seq(
+ // We don't have project on tuples in the Scala API
+ """^org\.apache\.flink\.api.java.*project""",
+
+ // I don't want to have withParameters in the API since I consider Configuration to be
+ // deprecated. But maybe thats just me ...
+ """^org\.apache\.flink\.api.java.*withParameters""",
+
+ // These are only used internally. Should be internal API but Java doesn't have
+ // private[flink].
+ """^org\.apache\.flink\.api.java.*getBroadcastSets""",
+ """^org\.apache\.flink\.api.java.*setSemanticProperties""",
+ """^org\.apache\.flink\.api.java.*getSemanticProperties""",
+ """^org\.apache\.flink\.api.java.*getParameters""",
+
+ // Commented out for now until we have a use case for this.
+ """^org\.apache\.flink\.api.java.*runOperation""",
+
+ // Object methods
+ """^.*notify""",
+ """^.*wait""",
+ """^.*notifyAll""",
+ """^.*equals""",
+ """^.*toString""",
+ """^.*getClass""",
+ """^.*hashCode"""
+ ).map(_.r)
+ lazy val excludedByPattern =
+ excludedPatterns.map(_.findFirstIn(name)).filter(_.isDefined).nonEmpty
+ name.contains("$") || excludedNames.contains(name) || excludedByPattern
+ }
+
+ private def isExcludedByInterface(method: Method): Boolean = {
+ val excludedInterfaces =
+ Set("org.apache.spark.Logging", "org.apache.hadoop.mapreduce.HadoopMapReduceUtil")
+ def toComparisionKey(method: Method) =
+ (method.getReturnType, method.getName, method.getGenericReturnType)
+ val interfaces = method.getDeclaringClass.getInterfaces.filter { i =>
+ excludedInterfaces.contains(i.getName)
+ }
+ val excludedMethods = interfaces.flatMap(_.getMethods.map(toComparisionKey))
+ excludedMethods.contains(toComparisionKey(method))
+ }
+
+ private def checkMethods(
+ javaClassName: String,
+ scalaClassName: String,
+ javaClass: Class[_],
+ scalaClass: Class[_]) {
+ val javaMethods = javaClass.getMethods
+ .filterNot(_.isAccessible)
+ .filterNot(isExcludedByName)
+ .filterNot(isExcludedByInterface)
+ .map(m => m.getName).toSet
+
+ val scalaMethods = scalaClass.getMethods
+ .filterNot(_.isAccessible)
+ .filterNot(isExcludedByName)
+ .filterNot(isExcludedByInterface)
+ .map(m => m.getName).toSet
+
+ val missingMethods = javaMethods -- scalaMethods
+
+ for (method <- missingMethods) {
+ fail("Method " + method + " from " + javaClass + " is missing from " + scalaClassName + ".")
+ }
+ }
+
+ @Test
+ def testCompleteness(): Unit = {
+ checkMethods("DataSet", "DataSet", classOf[JavaDataSet[_]], classOf[DataSet[_]])
+
+ checkMethods(
+ "ExecutionEnvironment", "ExecutionEnvironment",
+ classOf[org.apache.flink.api.java.ExecutionEnvironment], classOf[ExecutionEnvironment])
+
+ checkMethods("Operator", "DataSet", classOf[Operator[_, _]], classOf[DataSet[_]])
+
+ checkMethods("UnsortedGrouping", "GroupedDataSet",
+ classOf[UnsortedGrouping[_]], classOf[GroupedDataSet[_]])
+
+ checkMethods("SortedGrouping", "GroupedDataSet",
+ classOf[SortedGrouping[_]], classOf[GroupedDataSet[_]])
+
+ checkMethods("AggregateOperator", "AggregateDataSet",
+ classOf[AggregateOperator[_]], classOf[AggregateDataSet[_]])
+
+ checkMethods("SingleInputOperator", "DataSet",
+ classOf[SingleInputOperator[_, _, _]], classOf[DataSet[_]])
+
+ checkMethods("TwoInputOperator", "DataSet",
+ classOf[TwoInputOperator[_, _, _, _]], classOf[DataSet[_]])
+
+ checkMethods("SingleInputUdfOperator", "DataSet",
+ classOf[SingleInputUdfOperator[_, _, _]], classOf[DataSet[_]])
+
+ checkMethods("TwoInputUdfOperator", "DataSet",
+ classOf[TwoInputUdfOperator[_, _, _, _]], classOf[DataSet[_]])
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
new file mode 100644
index 0000000..543c05a
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.functions
+
+import org.junit.Assert._
+import org.apache.flink.api.common.functions.RichJoinFunction
+import org.apache.flink.api.common.functions.RichMapFunction
+import org.apache.flink.api.common.operators.{GenericDataSinkBase, SingleInputSemanticProperties}
+import org.apache.flink.api.common.operators.base.{JoinOperatorBase, MapOperatorBase}
+import org.apache.flink.api.common.operators.util.FieldSet
+import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields
+import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsFirst
+import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsSecond
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+/**
+ * This is a minimal test to verify that semantic annotations are evaluated against
+ * the type information properly translated correctly to the common data flow API.
+ *
+ * This covers only the constant fields annotations currently !!!
+ */
+class SemanticPropertiesTranslationTest {
+ /**
+ * A mapper that preserves all fields over a tuple data set.
+ */
+ @Test
+ def translateUnaryFunctionAnnotationTuplesWildCard(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val input = env.fromElements((3L, "test", 42))
+ input.map(new WildcardConstantMapper[(Long, String, Int)]).print()
+
+ val plan = env.createProgramPlan()
+
+ val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
+
+ val mapper: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_, _, _]]
+
+ val semantics: SingleInputSemanticProperties = mapper.getSemanticProperties
+ val fw1: FieldSet = semantics.getForwardedField(0)
+ val fw2: FieldSet = semantics.getForwardedField(1)
+ val fw3: FieldSet = semantics.getForwardedField(2)
+
+ assertNotNull(fw1)
+ assertNotNull(fw2)
+ assertNotNull(fw3)
+ assertTrue(fw1.contains(0))
+ assertTrue(fw2.contains(1))
+ assertTrue(fw3.contains(2))
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+
+ /**
+ * A mapper that preserves fields 0, 1, 2 of a tuple data set.
+ */
+ @Test
+ def translateUnaryFunctionAnnotationTuples(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val input = env.fromElements((3L, "test", 42))
+ input.map(new IndividualConstantMapper[Long, String, Int]).print()
+
+ val plan = env.createProgramPlan()
+
+ val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
+
+ val mapper: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_, _, _]]
+
+ val semantics: SingleInputSemanticProperties = mapper.getSemanticProperties
+ val fw1: FieldSet = semantics.getForwardedField(0)
+ val fw2: FieldSet = semantics.getForwardedField(1)
+ val fw3: FieldSet = semantics.getForwardedField(2)
+
+ assertNotNull(fw1)
+ assertNotNull(fw2)
+ assertNotNull(fw3)
+ assertTrue(fw1.contains(0))
+ assertTrue(fw2.contains(1))
+ assertTrue(fw3.contains(2))
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+
+ /**
+ * A join that preserves tuple fields from both sides.
+ */
+ @Test
+ def translateBinaryFunctionAnnotationTuples(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val input1 = env.fromElements((3L, "test"))
+ val input2 = env.fromElements((3L, 3.1415))
+
+ input1.join(input2).where(0).equalTo(0)(
+ new ForwardingTupleJoin[Long, String, Long, Double]).print()
+
+ val plan = env.createProgramPlan()
+ val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
+
+ val join: JoinOperatorBase[_, _, _, _] =
+ sink.getInput.asInstanceOf[JoinOperatorBase[_, _, _, _]]
+
+ val semantics = join.getSemanticProperties
+ val fw11: FieldSet = semantics.getForwardedField1(0)
+ val fw12: FieldSet = semantics.getForwardedField1(1)
+ val fw21: FieldSet = semantics.getForwardedField2(0)
+ val fw22: FieldSet = semantics.getForwardedField2(1)
+
+ assertNull(fw11)
+ assertNull(fw21)
+ assertNotNull(fw12)
+ assertNotNull(fw22)
+ assertTrue(fw12.contains(0))
+ assertTrue(fw22.contains(1))
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+}
+
+
+@ConstantFields(Array("*"))
+class WildcardConstantMapper[T] extends RichMapFunction[T, T] {
+ def map(value: T): T = {
+ value
+ }
+}
+
+@ConstantFields(Array("0->0;1->1;2->2"))
+class IndividualConstantMapper[X, Y, Z] extends RichMapFunction[(X, Y, Z), (X, Y, Z)] {
+ def map(value: (X, Y, Z)): (X, Y, Z) = {
+ value
+ }
+}
+
+@ConstantFields(Array("0"))
+class ZeroConstantMapper[T] extends RichMapFunction[T, T] {
+ def map(value: T): T = {
+ value
+ }
+}
+
+@ConstantFieldsFirst(Array("1 -> 0"))
+@ConstantFieldsSecond(Array("1 -> 1"))
+class ForwardingTupleJoin[A, B, C, D] extends RichJoinFunction[(A, B), (C, D), (B, D)] {
+ def join(first: (A, B), second: (C, D)): (B, D) = {
+ (first._2, second._2)
+ }
+}
+
+@ConstantFieldsFirst(Array("0 -> 0"))
+@ConstantFieldsSecond(Array("0 -> 1"))
+class ForwardingBasicJoin[A, B] extends RichJoinFunction[A, B, (A, B)] {
+ def join(first: A, second: B): (A, B) = {
+ (first, second)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
new file mode 100644
index 0000000..a81dc45
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.io
+
+import org.apache.flink.api.java.io.CollectionInputFormat
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertTrue
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.core.io.GenericInputSplit
+import org.junit.Test
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.ObjectInputStream
+import java.io.ObjectOutputStream
+import org.apache.flink.api.scala._
+import scala.collection.JavaConverters._
+
+class ElementType(val id: Int) {
+ def this() {
+ this(-1)
+ }
+
+ override def equals(obj: Any): Boolean = {
+ if (obj != null && obj.isInstanceOf[ElementType]) {
+ val et = obj.asInstanceOf[ElementType]
+ et.id == this.id
+ }
+ else {
+ false
+ }
+ }
+}
+
+class CollectionInputFormatTest {
+
+ @Test
+ def testSerializability(): Unit = {
+
+ val inputCollection = Seq(new ElementType(1), new ElementType(2), new ElementType(3))
+ val info = createTypeInformation[ElementType]
+
+ val inputFormat: CollectionInputFormat[ElementType] = new
+ CollectionInputFormat[ElementType](inputCollection.asJava, info.createSerializer())
+
+ val buffer = new ByteArrayOutputStream
+ val out = new ObjectOutputStream(buffer)
+
+ out.writeObject(inputFormat)
+
+ val in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray))
+ val serializationResult: AnyRef = in.readObject
+
+ assertNotNull(serializationResult)
+ assertTrue(serializationResult.isInstanceOf[CollectionInputFormat[_]])
+
+ val result = serializationResult.asInstanceOf[CollectionInputFormat[ElementType]]
+ val inputSplit = new GenericInputSplit
+ inputFormat.open(inputSplit)
+ result.open(inputSplit)
+
+ while (!inputFormat.reachedEnd && !result.reachedEnd) {
+ val expectedElement = inputFormat.nextRecord(null)
+ val actualElement = result.nextRecord(null)
+ assertEquals(expectedElement, actualElement)
+ }
+ }
+
+ @Test
+ def testSerializabilityStrings(): Unit = {
+ val data = Seq("To bey or not to be,--that is the question:--",
+ "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune",
+ "Or to take arms against a sea of troubles,", "And by opposing end them?--To die," +
+ "--to sleep,--", "No more; and by a sleep to say we end", "The heartache, " +
+ "and the thousand natural shocks", "That flesh is heir to,--'tis a consummation",
+ "Devoutly to be wish'd. To die,--to sleep;--", "To sleep! perchance to dream:--ay, " +
+ "there's the rub;", "For in that sleep of death what dreams may come,",
+ "When we have shuffled off this mortal coil,", "Must give us pause: there's the respect",
+ "That makes calamity of so long life;", "For who would bear the whips and scorns of time,",
+ "The oppressor's wrong, the proud man's contumely,", "The pangs of despis'd love, " +
+ "the law's delay,", "The insolence of office, and the spurns",
+ "That patient merit of the unworthy takes,", "When he himself might his quietus make",
+ "With a bare bodkin? who would these fardels bear,", "To grunt and sweat under a weary " +
+ "life,", "But that the dread of something after death,--", "The undiscover'd country, " +
+ "from whose bourn", "No traveller returns,--puzzles the will,",
+ "And makes us rather bear those ills we have", "Than fly to others that we know not of?",
+ "Thus conscience does make cowards of us all;", "And thus the native hue of resolution",
+ "Is sicklied o'er with the pale cast of thought;", "And enterprises of great pith and " +
+ "moment,", "With this regard, their currents turn awry,", "And lose the name of action" +
+ ".--Soft you now!", "The fair Ophelia!--Nymph, in thy orisons",
+ "Be all my sins remember'd.")
+
+ val inputFormat = new CollectionInputFormat[String](
+ data.asJava,
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer)
+ val baos = new ByteArrayOutputStream
+ val oos = new ObjectOutputStream(baos)
+
+ oos.writeObject(inputFormat)
+ oos.close()
+
+ val bais = new ByteArrayInputStream(baos.toByteArray)
+ val ois = new ObjectInputStream(bais)
+ val result: AnyRef = ois.readObject
+
+ assertTrue(result.isInstanceOf[CollectionInputFormat[_]])
+ var i: Int = 0
+ val in = result.asInstanceOf[CollectionInputFormat[String]]
+ in.open(new GenericInputSplit)
+
+ while (!in.reachedEnd) {
+ assertEquals(data(i), in.nextRecord(""))
+ i += 1
+ }
+ assertEquals(data.length, i)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
new file mode 100644
index 0000000..9c90788
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.io
+
+import org.apache.flink.api.scala.operators.ScalaCsvInputFormat
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
+import java.io.File
+import java.io.FileOutputStream
+import java.io.FileWriter
+import java.io.OutputStreamWriter
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.core.fs.FileInputSplit
+import org.apache.flink.core.fs.Path
+import org.junit.Test
+import org.apache.flink.api.scala._
+
+class CsvInputFormatTest {
+
+ private final val PATH: Path = new Path("an/ignored/file/")
+ private final val FIRST_PART: String = "That is the first part"
+ private final val SECOND_PART: String = "That is the second part"
+
+ @Test
+ def readStringFields():Unit = {
+ try {
+ val fileContent = "abc|def|ghijk\nabc||hhg\n|||"
+ val split = createTempFile(fileContent)
+ val format = new ScalaCsvInputFormat[(String, String, String)](
+ PATH, createTypeInformation[(String, String, String)])
+ format.setDelimiter("\n")
+ format.setFieldDelimiter('|')
+ val parameters = new Configuration
+ format.configure(parameters)
+ format.open(split)
+ var result: (String, String, String) = null
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("abc", result._1)
+ assertEquals("def", result._2)
+ assertEquals("ghijk", result._3)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("abc", result._1)
+ assertEquals("", result._2)
+ assertEquals("hhg", result._3)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("", result._1)
+ assertEquals("", result._2)
+ assertEquals("", result._3)
+ result = format.nextRecord(result)
+ assertNull(result)
+ assertTrue(format.reachedEnd)
+ }
+ catch {
+ case ex: Exception => {
+ ex.printStackTrace()
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def readStringFieldsWithTrailingDelimiters(): Unit = {
+ try {
+ val fileContent = "abc|def|ghijk\nabc||hhg\n|||\n"
+ val split = createTempFile(fileContent)
+ val format = new ScalaCsvInputFormat[(String, String, String)](
+ PATH, createTypeInformation[(String, String, String)])
+ format.setDelimiter("\n")
+ format.setFieldDelimiter('|')
+ val parameters = new Configuration
+ format.configure(parameters)
+ format.open(split)
+ var result: (String, String, String) = null
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("abc", result._1)
+ assertEquals("def", result._2)
+ assertEquals("ghijk", result._3)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("abc", result._1)
+ assertEquals("", result._2)
+ assertEquals("hhg", result._3)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals("", result._1)
+ assertEquals("", result._2)
+ assertEquals("", result._3)
+ result = format.nextRecord(result)
+ assertNull(result)
+ assertTrue(format.reachedEnd)
+ }
+ catch {
+ case ex: Exception =>
+ ex.printStackTrace()
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+
+ @Test
+ def testIntegerFields(): Unit = {
+ try {
+ val fileContent = "111|222|333|444|555\n666|777|888|999|000|\n"
+ val split = createTempFile(fileContent)
+ val format = new ScalaCsvInputFormat[(Int, Int, Int, Int, Int)](
+ PATH, createTypeInformation[(Int, Int, Int, Int, Int)])
+ format.setFieldDelimiter('|')
+ format.configure(new Configuration)
+ format.open(split)
+ var result: (Int, Int, Int, Int, Int) = null
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(111), result._1)
+ assertEquals(Integer.valueOf(222), result._2)
+ assertEquals(Integer.valueOf(333), result._3)
+ assertEquals(Integer.valueOf(444), result._4)
+ assertEquals(Integer.valueOf(555), result._5)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(666), result._1)
+ assertEquals(Integer.valueOf(777), result._2)
+ assertEquals(Integer.valueOf(888), result._3)
+ assertEquals(Integer.valueOf(999), result._4)
+ assertEquals(Integer.valueOf(000), result._5)
+ result = format.nextRecord(result)
+ assertNull(result)
+ assertTrue(format.reachedEnd)
+ }
+ catch {
+ case ex: Exception =>
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+
+ @Test
+ def testReadFirstN(): Unit = {
+ try {
+ val fileContent = "111|222|333|444|555|\n666|777|888|999|000|\n"
+ val split = createTempFile(fileContent)
+ val format = new ScalaCsvInputFormat[(Int, Int)](PATH, createTypeInformation[(Int, Int)])
+ format.setFieldDelimiter('|')
+ format.configure(new Configuration)
+ format.open(split)
+ var result: (Int, Int) = null
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(111), result._1)
+ assertEquals(Integer.valueOf(222), result._2)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(666), result._1)
+ assertEquals(Integer.valueOf(777), result._2)
+ result = format.nextRecord(result)
+ assertNull(result)
+ assertTrue(format.reachedEnd)
+ }
+ catch {
+ case ex: Exception =>
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+
+ @Test
+ def testReadSparseWithPositionSetter(): Unit = {
+ try {
+ val fileContent: String = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666" +
+ "|555|444|333|222|111|"
+ val split = createTempFile(fileContent)
+ val format = new ScalaCsvInputFormat[(Int, Int, Int)](
+ PATH,
+ createTypeInformation[(Int, Int, Int)])
+ format.setFieldDelimiter('|')
+ format.setFields(Array(0, 3, 7), Array(classOf[Integer], classOf[Integer], classOf[Integer]))
+ format.configure(new Configuration)
+ format.open(split)
+ var result: (Int, Int, Int) = null
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(111), result._1)
+ assertEquals(Integer.valueOf(444), result._2)
+ assertEquals(Integer.valueOf(888), result._3)
+ result = format.nextRecord(result)
+ assertNotNull(result)
+ assertEquals(Integer.valueOf(000), result._1)
+ assertEquals(Integer.valueOf(777), result._2)
+ assertEquals(Integer.valueOf(333), result._3)
+ result = format.nextRecord(result)
+ assertNull(result)
+ assertTrue(format.reachedEnd)
+ }
+ catch {
+ case ex: Exception =>
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+
+ @Test
+ def testReadSparseWithShuffledPositions(): Unit = {
+ try {
+ val format = new ScalaCsvInputFormat[(Int, Int, Int)](
+ PATH,
+ createTypeInformation[(Int, Int, Int)])
+ format.setFieldDelimiter('|')
+ try {
+ format.setFields(Array(8, 1, 3), Array(classOf[Integer],classOf[Integer],classOf[Integer]))
+ fail("Input sequence should have been rejected.")
+ }
+ catch {
+ case e: IllegalArgumentException => // ignore
+ }
+ }
+ catch {
+ case ex: Exception =>
+ fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
+ }
+ }
+
+ private def createTempFile(content: String): FileInputSplit = {
+ val tempFile = File.createTempFile("test_contents", "tmp")
+ tempFile.deleteOnExit()
+ val wrt = new FileWriter(tempFile)
+ wrt.write(content)
+ wrt.close()
+ new FileInputSplit(0, new Path(tempFile.toURI.toString), 0,
+ tempFile.length,Array[String]("localhost"))
+ }
+
+ @Test
+ def testWindowsLineEndRemoval(): Unit = {
+ this.testRemovingTrailingCR("\n", "\n")
+ this.testRemovingTrailingCR("\r\n", "\r\n")
+ this.testRemovingTrailingCR("\r\n", "\n")
+ }
+
+ private def testRemovingTrailingCR(lineBreakerInFile: String, lineBreakerSetup: String) {
+ var tempFile: File = null
+ val fileContent = FIRST_PART + lineBreakerInFile + SECOND_PART + lineBreakerInFile
+ try {
+ tempFile = File.createTempFile("CsvInputFormatTest", "tmp")
+ tempFile.deleteOnExit()
+ tempFile.setWritable(true)
+ val wrt = new OutputStreamWriter(new FileOutputStream(tempFile))
+ wrt.write(fileContent)
+ wrt.close()
+ val inputFormat = new ScalaCsvInputFormat[Tuple1[String]](new Path(tempFile.toURI.toString),
+ createTypeInformation[Tuple1[String]])
+ val parameters = new Configuration
+ inputFormat.configure(parameters)
+ inputFormat.setDelimiter(lineBreakerSetup)
+ val splits = inputFormat.createInputSplits(1)
+ inputFormat.open(splits(0))
+ var result = inputFormat.nextRecord(null)
+ assertNotNull("Expecting to not return null", result)
+ assertEquals(FIRST_PART, result._1)
+ result = inputFormat.nextRecord(result)
+ assertNotNull("Expecting to not return null", result)
+ assertEquals(SECOND_PART, result._1)
+ }
+ catch {
+ case t: Throwable =>
+ System.err.println("test failed with exception: " + t.getMessage)
+ t.printStackTrace(System.err)
+ fail("Test erroneous")
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
new file mode 100644
index 0000000..ad22fa6
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.java.aggregation.Aggregations
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object AggregateProgs {
+ var NUM_PROGRAMS: Int = 3
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ // Full aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(10)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .aggregate(Aggregations.SUM,0)
+ .and(Aggregations.MAX, 1)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map{ t => (t._1, t._2) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "231,6\n"
+
+ case 2 =>
+ // Grouped aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .groupBy(1)
+ .aggregate(Aggregations.SUM, 0)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map { t => (t._2, t._1) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n"
+
+ case 3 =>
+ // Nested aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .groupBy(1)
+ .aggregate(Aggregations.MIN, 0)
+ .aggregate(Aggregations.MIN, 0)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map { t => new Tuple1(t._1) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1\n"
+
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class AggregateITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = AggregateProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object AggregateITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to AggregateProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
new file mode 100644
index 0000000..639884b
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.junit.Assert
+import org.apache.flink.api.common.InvalidProgramException
+import org.apache.flink.api.java.aggregation.Aggregations
+import org.apache.flink.api.java.aggregation.UnsupportedAggregationTypeException
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class AggregateOperatorTest {
+
+ private final val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+ private final val tupleTypeInfo = createTypeInformation[(Int, Long, String, Long, Int)]
+ private final val emptyLongData = Array[Long]()
+
+ @Test
+ def testFieldsAggregate(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ tupleDs.aggregate(Aggregations.SUM, 1)
+ } catch {
+ case e: Exception => Assert.fail()
+ }
+
+ // should not work: index out of bounds
+ try {
+ tupleDs.aggregate(Aggregations.SUM, 10)
+ Assert.fail()
+ } catch {
+ case iae: IllegalArgumentException =>
+ case e: Exception => Assert.fail()
+ }
+
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work: not applied to tuple DataSet
+ try {
+ longDs.aggregate(Aggregations.MIN, 1)
+ Assert.fail()
+ } catch {
+ case uoe: InvalidProgramException =>
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testFieldNamesAggregate(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ tupleDs.aggregate(Aggregations.SUM, "_2")
+ } catch {
+ case e: Exception => Assert.fail()
+ }
+
+ // should not work: invalid field
+ try {
+ tupleDs.aggregate(Aggregations.SUM, "foo")
+ Assert.fail()
+ } catch {
+ case iae: IllegalArgumentException =>
+ case e: Exception => Assert.fail()
+ }
+
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work: not applied to tuple DataSet
+ try {
+ longDs.aggregate(Aggregations.MIN, "_1")
+ Assert.fail()
+ } catch {
+ case uoe: InvalidProgramException =>
+ case uoe: UnsupportedOperationException =>
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testAggregationTypes(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work: multiple aggregates
+ tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4)
+
+ // should work: nested aggregates
+ tupleDs.aggregate(Aggregations.MIN, 2).and(Aggregations.SUM, 1)
+
+ // should not work: average on string
+ try {
+ tupleDs.aggregate(Aggregations.SUM, 2)
+ Assert.fail()
+ } catch {
+ case iae: UnsupportedAggregationTypeException =>
+ }
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ Assert.fail(e.getMessage)
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
new file mode 100644
index 0000000..d4ce3b7
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichCoGroupFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+import org.junit.Assert
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object CoGroupProgs {
+ var NUM_PROGRAMS: Int = 13
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * CoGroup on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
+ (first, second) =>
+ var sum = 0
+ var id = 0
+ for (t <- first) {
+ sum += t._3
+ id = t._1
+ }
+ for (t <- second) {
+ sum += t._3
+ id = t._1
+ }
+ (id, sum)
+ }
+ coGroupDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,0\n" + "2,6\n" + "3,24\n" + "4,60\n" + "5,120\n"
+
+ case 2 =>
+ /*
+ * CoGroup on two custom type inputs with key extractors
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
+
+ val coGroupDs = ds.coGroup(ds2).where(_.myInt).equalTo(_.myInt) apply {
+ (first, second) =>
+ val o = new CustomType(0, 0, "test")
+ for (c <- first) {
+ o.myInt = c.myInt
+ o.myLong += c.myLong
+ }
+ for (c <- second) {
+ o.myInt = c.myInt
+ o.myLong += c.myLong
+ }
+ o
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,test\n" + "2,6,test\n" + "3,24,test\n" + "4,60,test\n" + "5,120,test\n" + "6," +
+ "210,test\n"
+
+ case 3 =>
+ /*
+ * check correctness of cogroup if UDF returns left input objects
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get3TupleDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
+ (first, second, out: Collector[(Int, Long, String)] ) =>
+ for (t <- first) {
+ if (t._1 < 6) {
+ out.collect(t)
+ }
+ }
+ }
+ coGroupDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
+ "how are you?\n" + "5,3,I am fine.\n"
+
+ case 4 =>
+ /*
+ * check correctness of cogroup if UDF returns right input objects
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
+ (first, second, out: Collector[(Int, Long, Int, String, Long)]) =>
+ for (t <- second) {
+ if (t._1 < 4) {
+ out.collect(t)
+ }
+ }
+ }
+ coGroupDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt,2\n" + "2,3,2,Hallo Welt wie,1\n" + "3,4,3," +
+ "Hallo Welt wie gehts?,2\n" + "3,5,4,ABC,2\n" + "3,6,5,BCD,3\n"
+
+ case 5 =>
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).apply(
+ new RichCoGroupFunction[
+ (Int, Long, Int, String, Long),
+ (Int, Long, Int, String, Long),
+ (Int, Int, Int)] {
+ private var broadcast = 41
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ broadcast = ints.sum
+ }
+
+ override def coGroup(
+ first: java.lang.Iterable[(Int, Long, Int, String, Long)],
+ second: java.lang.Iterable[(Int, Long, Int, String, Long)],
+ out: Collector[(Int, Int, Int)]): Unit = {
+ var sum = 0
+ var id = 0
+ for (t <- first.asScala) {
+ sum += t._3
+ id = t._1
+ }
+ for (t <- second.asScala) {
+ sum += t._3
+ id = t._1
+ }
+ out.collect((id, sum, broadcast))
+ }
+
+ }).withBroadcastSet(intDs, "ints")
+ coGroupDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,0,55\n" + "2,6,55\n" + "3,24,55\n" + "4,60,55\n" + "5,120,55\n"
+
+ case 6 =>
+ /*
+ * CoGroup on a tuple input with key field selector and a custom type input with
+ * key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(2).equalTo(_.myInt) apply {
+ (first, second) =>
+ var sum = 0L
+ var id = 0
+ for (t <- first) {
+ sum += t._1
+ id = t._3
+ }
+ for (t <- second) {
+ sum += t.myLong
+ id = t.myInt
+ }
+ (id, sum, "test")
+ }
+ coGroupDs.writeAsCsv(resultPath)
+ env.execute()
+ "0,1,test\n" + "1,2,test\n" + "2,5,test\n" + "3,15,test\n" + "4,33,test\n" + "5," +
+ "63,test\n" + "6,109,test\n" + "7,4,test\n" + "8,4,test\n" + "9,4,test\n" + "10,5," +
+ "test\n" + "11,5,test\n" + "12,5,test\n" + "13,5,test\n" + "14,5,test\n"
+
+ case 7 =>
+ /*
+ * CoGroup on a tuple input with key field selector and a custom type input with
+ * key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
+ val coGroupDs = ds2.coGroup(ds).where(_.myInt).equalTo(2) {
+ (first, second) =>
+ var sum = 0L
+ var id = 0
+ for (t <- first) {
+ sum += t.myLong
+ id = t.myInt
+ }
+ for (t <- second) {
+ sum += t._1
+ id = t._3
+ }
+
+ new CustomType(id, sum, "test")
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "0,1,test\n" + "1,2,test\n" + "2,5,test\n" + "3,15,test\n" + "4,33,test\n" + "5," +
+ "63,test\n" + "6,109,test\n" + "7,4,test\n" + "8,4,test\n" + "9,4,test\n" + "10,5," +
+ "test\n" + "11,5,test\n" + "12,5,test\n" + "13,5,test\n" + "14,5,test\n"
+
+ case 8 =>
+ /*
+ * CoGroup with multiple key fields
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.get3TupleDataSet(env)
+ val coGrouped = ds1.coGroup(ds2).where(0,4).equalTo(0, 1) {
+ (first, second, out: Collector[(Int, Long, String)]) =>
+ val strs = first map(_._4)
+ for (t <- second) {
+ for (s <- strs) {
+ out.collect((t._1, t._2, s))
+ }
+ }
+ }
+
+ coGrouped.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hallo\n" + "2,2,Hallo Welt\n" + "3,2,Hallo Welt wie gehts?\n" + "3,2," +
+ "ABC\n" + "5,3,HIJ\n" + "5,3,IJK\n"
+
+ case 9 =>
+ /*
+ * CoGroup with multiple key fields
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets
+ .get5TupleDataSet(env)
+ val ds2 = CollectionDataSets.get3TupleDataSet(env)
+ val coGrouped = ds1.coGroup(ds2).where(t => (t._1, t._5)).equalTo(t => (t._1, t._2))
+ .apply {
+ (first, second, out: Collector[(Int, Long, String)]) =>
+ val strs = first map(_._4)
+ for (t <- second) {
+ for (s <- strs) {
+ out.collect((t._1, t._2, s))
+ }
+ }
+ }
+
+ coGrouped.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hallo\n" + "2,2,Hallo Welt\n" + "3,2,Hallo Welt wie gehts?\n" + "3,2," +
+ "ABC\n" + "5,3,HIJ\n" + "5,3,IJK\n"
+
+ case 10 =>
+ /*
+ * CoGroup on two custom type inputs using expression keys
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where("myInt").equalTo("myInt") {
+ (first, second) =>
+ val o = new CustomType(0, 0, "test")
+ for (t <- first) {
+ o.myInt = t.myInt
+ o.myLong += t.myLong
+ }
+ for (t <- second) {
+ o.myInt = t.myInt
+ o.myLong += t.myLong
+ }
+ o
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,test\n" + "2,6,test\n" + "3,24,test\n" + "4,60,test\n" + "5,120,test\n" + "6," +
+ "210,test\n"
+
+ case 11 =>
+ /*
+ * CoGroup on two custom type inputs using expression keys
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where("nestedPojo.longNumber").equalTo(6) {
+ (first, second, out: Collector[CustomType]) =>
+ for (p <- first) {
+ for (t <- second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t._7)
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
+ }
+ }
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
+
+ case 12 =>
+ /*
+ * CoGroup field-selector (expression keys) + key selector function
+ * The key selector is unnecessary complicated (Tuple1) ;)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(t => new Tuple1(t.nestedPojo.longNumber)).equalTo(6) {
+ (first, second, out: Collector[CustomType]) =>
+ for (p <- first) {
+ for (t <- second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t._7)
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
+ }
+ }
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
+
+ case 13 =>
+ /*
+ * CoGroup field-selector (expression keys) + key selector function
+ * The key selector is simple here
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val coGroupDs = ds.coGroup(ds2).where(_.nestedPojo.longNumber).equalTo(6) {
+ (first, second, out: Collector[CustomType]) =>
+ for (p <- first) {
+ for (t <- second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t._7)
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
+ }
+ }
+ }
+ coGroupDs.writeAsText(resultPath)
+ env.execute()
+ "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class CoGroupITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private val curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = CoGroupProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object CoGroupITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to CoGroupProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
new file mode 100644
index 0000000..3f0ca5f
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
+import org.junit.Assert
+import org.junit.Test
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+
+class CoGroupOperatorTest {
+
+ private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+ private var customTypeData = Array[CustomType](new CustomType())
+
+ @Test
+ def testCoGroupKeyFields1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should work
+ try {
+ ds1.coGroup(ds2).where(0).equalTo(0)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyFields2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, incompatible key types
+ ds1.coGroup(ds2).where(0).equalTo(2)
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyFields3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, incompatible number of key fields
+ ds1.coGroup(ds2).where(0, 1).equalTo(2)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testCoGroupKeyFields4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, field position out of range
+ ds1.coGroup(ds2).where(5).equalTo(0)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testCoGroupKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, negative field position
+ ds1.coGroup(ds2).where(-1).equalTo(-1)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testCoGroupKeyFields6(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should not work, field position key on custom data type
+ ds1.coGroup(ds2).where(5).equalTo(0)
+ }
+
+ @Test
+ def testCoGroupKeyFieldNames1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should work
+ try {
+ ds1.coGroup(ds2).where("_1").equalTo("_1")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyFieldNames2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, incompatible key types
+ ds1.coGroup(ds2).where("_1").equalTo("_3")
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyFieldNames3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, incompatible number of key fields
+ ds1.coGroup(ds2).where("_1", "_2").equalTo("_3")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testCoGroupKeyFieldNames4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, invalid field name
+ ds1.coGroup(ds2).where("_6").equalTo("_1")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testCoGroupKeyFieldNames5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should not work, invalid field name
+ ds1.coGroup(ds2).where("_1").equalTo("bar")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testCoGroupKeyFieldNames6(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should not work, field position key on custom data type
+ ds1.coGroup(ds2).where("_3").equalTo("_1")
+ }
+
+ @Test
+ def testCoGroupKeyExpressions1(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should work
+ try {
+// ds1.coGroup(ds2).where("i").equalTo("i");
+
+ }catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyExpressions2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, incompatible key types
+ ds1.coGroup(ds2).where("myInt").equalTo("myString")
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyExpressions3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, incompatible number of keys
+ ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString")
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testCoGroupKeyExpressions4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+
+ // should not work, key non-existent
+ ds1.coGroup(ds2).where("myNonExistent").equalTo("i")
+ }
+
+ @Test
+ def testCoGroupKeySelectors1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should work
+ try {
+ ds1.coGroup(ds2).where { _.myLong } equalTo { _.myLong }
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testCoGroupKeyMixing1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // Should work
+ try {
+ ds1.coGroup(ds2).where { _.myLong }.equalTo(3)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testCoGroupKeyMixing2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should work
+ try {
+ ds1.coGroup(ds2).where(3).equalTo { _.myLong }
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyMixing3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should not work, incompatible types
+ ds1.coGroup(ds2).where(2).equalTo { _.myLong }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testCoGroupKeyMixing4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // Should not work, more than one field position key
+ ds1.coGroup(ds2).where(1, 3).equalTo { _.myLong }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
new file mode 100644
index 0000000..bcb85e5
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichCrossFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object CrossProgs {
+ var NUM_PROGRAMS: Int = 9
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * check correctness of cross on two tuple inputs
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.cross(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4) }
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+
+ "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
+ "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
+ "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
+ "Hallo Welt wieHallo Welt wie\n"
+
+ case 2 =>
+ /*
+ * check correctness of cross if UDF returns left input object
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.cross(ds2) { (l, r ) => l }
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+
+ "1,1,Hi\n" + "1,1,Hi\n" + "1,1,Hi\n" + "2,2,Hello\n" + "2,2,Hello\n" + "2,2," +
+ "Hello\n" + "3,2,Hello world\n" + "3,2,Hello world\n" + "3,2,Hello world\n"
+
+ case 3 =>
+ /*
+ * check correctness of cross if UDF returns right input object
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.cross(ds2) { (l, r) => r }
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+
+ "1,1,0,Hallo,1\n" + "1,1,0,Hallo,1\n" + "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt," +
+ "2\n" + "2,2,1,Hallo Welt,2\n" + "2,2,1,Hallo Welt,2\n" + "2,3,2,Hallo Welt wie," +
+ "1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,3,2,Hallo Welt wie,1\n"
+
+ case 4 =>
+ /*
+ * check correctness of cross with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.cross(ds2).apply (
+ new RichCrossFunction[
+ (Int, Long, Int, String, Long),
+ (Int, Long, Int, String, Long),
+ (Int, Int, Int)] {
+ private var broadcast = 41
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ broadcast = ints.sum
+ }
+
+ override def cross(
+ first: (Int, Long, Int, String, Long),
+ second: (Int, Long, Int, String, Long)): (Int, Int, Int) = {
+ (first._1 + second._1, first._3.toInt * second._3.toInt, broadcast)
+ }
+
+ })withBroadcastSet(intDs, "ints")
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+ "2,0,55\n" + "3,0,55\n" + "3,0,55\n" + "3,0,55\n" + "4,1,55\n" + "4,2,55\n" + "3," +
+ "0,55\n" + "4,2,55\n" + "4,4,55\n"
+
+ case 5 =>
+ /*
+ * check correctness of crossWithHuge (only correctness of result -> should be the same
+ * as with normal cross)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.crossWithHuge(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4)}
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+ "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
+ "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
+ "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
+ "Hallo Welt wieHallo Welt wie\n"
+
+ case 6 =>
+ /*
+ * check correctness of crossWithTiny (only correctness of result -> should be the same
+ * as with normal cross)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .getSmall5TupleDataSet(env)
+ val ds2 = CollectionDataSets
+ .getSmall5TupleDataSet(env)
+ val crossDs = ds.crossWithTiny(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4)}
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+ "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
+ "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
+ "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
+ "Hallo Welt wieHallo Welt wie\n"
+
+ case 7 => // 9 in Java CrossITCase
+ /*
+ * check correctness of default cross
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val crossDs = ds.cross(ds2)
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+ "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" + "(1,1,Hi),(1,1,0,Hallo,1)\n" + "(1,1,Hi),(2,3," +
+ "2,Hallo Welt wie,1)\n" + "(2,2,Hello),(2,2,1,Hallo Welt,2)\n" + "(2,2,Hello),(1,1,0," +
+ "Hallo,1)\n" + "(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" + "(3,2,Hello world),(2,2,1," +
+ "Hallo Welt,2)\n" + "(3,2,Hello world),(1,1,0,Hallo,1)\n" + "(3,2,Hello world),(2,3,2," +
+ "Hallo Welt wie,1)\n"
+
+ case 8 => // 10 in Java CrossITCase
+ /*
+ * check correctness of cross on two custom type inputs
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmallCustomTypeDataSet(env)
+ val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
+ val crossDs = ds.cross(ds2) {
+ (l, r) => new CustomType(l.myInt * r.myInt, l.myLong + r.myLong, l.myString + r.myString)
+ }
+ crossDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,HiHi\n" + "2,1,HiHello\n" + "2,2,HiHello world\n" + "2,1,HelloHi\n" + "4,2," +
+ "HelloHello\n" + "4,3,HelloHello world\n" + "2,2,Hello worldHi\n" + "4,3," +
+ "Hello worldHello\n" + "4,4,Hello worldHello world"
+
+ case 9 => // 11 in Java CrossITCase
+ /*
+ * check correctness of cross a tuple input and a custom type input
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
+ val crossDs = ds.cross(ds2) {
+ (l, r) => (l._1 + r.myInt, l._3 * r.myLong, l._4 + r.myString)
+ }
+ crossDs.writeAsCsv(resultPath)
+ env.execute()
+ "2,0,HalloHi\n" + "3,0,HalloHello\n" + "3,0,HalloHello world\n" + "3,0," +
+ "Hallo WeltHi\n" + "4,1,Hallo WeltHello\n" + "4,2,Hallo WeltHello world\n" + "3,0," +
+ "Hallo Welt wieHi\n" + "4,2,Hallo Welt wieHello\n" + "4,4,Hallo Welt wieHello world\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class CrossITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = CrossProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object CrossITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to CrossProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
new file mode 100644
index 0000000..855335d
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object DistinctProgs {
+ var NUM_PROGRAMS: Int = 8
+
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Check correctness of distinct on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct(0, 1, 2)
+ distinctDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1,1,Hi\n" +
+ "2,2,Hello\n" +
+ "3,2,Hello world\n"
+
+ case 2 =>
+ /*
+ * check correctness of distinct on tuples with key field selector with not all fields
+ * selected
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct(0).map(_._1)
+
+ distinctDs.writeAsText(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 3 =>
+ /*
+ * check correctness of distinct on tuples with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val reduceDs = ds.union(ds).distinct(_._1).map(_._1)
+
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 4 =>
+ /*
+ * check correctness of distinct on custom type with type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+
+ val reduceDs = ds.distinct(_.myInt).map( t => new Tuple1(t.myInt))
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
+
+ case 5 =>
+ /*
+ * check correctness of distinct on tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct()
+
+ distinctDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n"
+
+ case 6 =>
+ /*
+ * check correctness of distinct on custom type with tuple-returning type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+
+ val reduceDs = ds.distinct( t => (t._1, t._5)).map( t => (t._1, t._5) )
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "2,1\n" + "2,2\n" + "3,2\n" + "3,3\n" + "4,1\n" + "4,2\n" + "5," +
+ "1\n" + "5,2\n" + "5,3\n"
+
+ case 7 =>
+ /*
+ * check correctness of distinct on tuples with field expressions
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val reduceDs = ds.union(ds).distinct("_1").map(t => new Tuple1(t._1))
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 8 =>
+ /*
+ * check correctness of distinct on Pojos
+ */
+
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
+
+ val reduceDs = ds.distinct("nestedPojo.longNumber").map(_.nestedPojo.longNumber.toInt)
+
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "10000\n20000\n30000\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class DistinctITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = DistinctProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object DistinctITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to DistinctProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
new file mode 100644
index 0000000..e9d214b
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.junit.Assert
+import org.apache.flink.api.common.InvalidProgramException
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class DistinctOperatorTest {
+
+ private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+ private val customTypeData = Array[CustomType](new CustomType())
+ private val emptyLongData = Array[Long]()
+
+ @Test
+ def testDistinctByKeyIndices1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // Should work
+ try {
+ tupleDs.distinct(0)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testDistinctByKeyIndices2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work: distinct on basic type
+ longDs.distinct(0)
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testDistinctByKeyIndices3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work: field position key on custom type
+ customDs.distinct(0)
+ }
+
+ @Test
+ def testDistinctByKeyIndices4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work
+ tupleDs.distinct()
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testDistinctByKeyIndices5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work: distinct on custom type
+ customDs.distinct()
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testDistinctByKeyIndices6(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, negative field position key
+ tupleDs.distinct(-1)
+ }
+
+ @Test
+ def testDistinctByKeyFields1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // Should work
+ try {
+ tupleDs.distinct("_1")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testDistinctByKeyFields2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work: distinct on basic type
+ longDs.distinct("_1")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testDistinctByKeyFields3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work: invalid fields
+ customDs.distinct("_1")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testDistinctByKeyFields4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, invalid field
+ tupleDs.distinct("foo")
+ }
+
+ @Test
+ def testDistinctByKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should work
+ customDs.distinct("myInt")
+ }
+
+ @Test
+ def testDistinctByKeySelector1(): Unit = {
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ try {
+ val customDs = env.fromCollection(customTypeData)
+ customDs.distinct {_.myLong}
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+}
+
[6/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
deleted file mode 100644
index 22e4914..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators.translation
-
-import org.apache.flink.api.common.operators.{GenericDataSourceBase, GenericDataSinkBase}
-import org.apache.flink.api.java.operators.translation.{KeyExtractingMapper,
-PlanUnwrappingReduceOperator}
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.java.typeutils.TupleTypeInfo
-import org.junit.Assert._
-import org.apache.flink.api.common.operators.base.MapOperatorBase
-import org.apache.flink.api.common.operators.base.ReduceOperatorBase
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class ReduceTranslationTest {
- @Test
- def translateNonGroupedReduce(): Unit = {
- try {
- val DOP = 8
- val env = ExecutionEnvironment.createLocalEnvironment(DOP)
-
- val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
-
-
- initialData reduce { (v1, v2) => v1 } print()
-
- val p = env.createProgramPlan(
-
-)
- val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
- val reducer: ReduceOperatorBase[_, _] = sink.getInput.asInstanceOf[ReduceOperatorBase[_, _]]
-
- assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getInputType)
- assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getOutputType)
- assertTrue(reducer.getKeyColumns(0) == null || reducer.getKeyColumns(0).length == 0)
- assertTrue(reducer.getDegreeOfParallelism == 1 || reducer.getDegreeOfParallelism == -1)
- assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Test caused an error: " + e.getMessage)
- }
- }
- }
-
- @Test
- def translateGroupedReduceNoMapper(): Unit = {
- try {
- val DOP: Int = 8
- val env = ExecutionEnvironment.createLocalEnvironment(DOP)
-
- val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
-
- initialData.groupBy(2) reduce { (v1, v2) => v1 } print()
-
- val p = env.createProgramPlan()
-
- val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
- val reducer: ReduceOperatorBase[_, _] = sink.getInput.asInstanceOf[ReduceOperatorBase[_, _]]
- assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getInputType)
- assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getOutputType)
- assertTrue(reducer.getDegreeOfParallelism == DOP || reducer.getDegreeOfParallelism == -1)
- assertArrayEquals(Array[Int](2), reducer.getKeyColumns(0))
- assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Test caused an error: " + e.getMessage)
- }
- }
- }
-
- @Test
- def translateGroupedReduceWithKeyExtractor(): Unit = {
- try {
- val DOP: Int = 8
- val env = ExecutionEnvironment.createLocalEnvironment(DOP)
-
- val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
-
- initialData.groupBy { _._2 }. reduce { (v1, v2) => v1 } setParallelism(4) print()
-
- val p = env.createProgramPlan()
- val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
- val keyProjector: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_,
- _, _]]
- val reducer: PlanUnwrappingReduceOperator[_, _] = keyProjector.getInput
- .asInstanceOf[PlanUnwrappingReduceOperator[_, _]]
- val keyExtractor: MapOperatorBase[_, _, _] = reducer.getInput
- .asInstanceOf[MapOperatorBase[_, _, _]]
- assertEquals(1, keyExtractor.getDegreeOfParallelism)
- assertEquals(4, reducer.getDegreeOfParallelism)
- assertEquals(4, keyProjector.getDegreeOfParallelism)
- val keyValueInfo = new TupleTypeInfo(
- BasicTypeInfo.STRING_TYPE_INFO,
- createTypeInformation[(Double, String, Long)])
- assertEquals(initialData.javaSet.getType, keyExtractor.getOperatorInfo.getInputType)
- assertEquals(keyValueInfo, keyExtractor.getOperatorInfo.getOutputType)
- assertEquals(keyValueInfo, reducer.getOperatorInfo.getInputType)
- assertEquals(keyValueInfo, reducer.getOperatorInfo.getOutputType)
- assertEquals(keyValueInfo, keyProjector.getOperatorInfo.getInputType)
- assertEquals(initialData.javaSet.getType, keyProjector.getOperatorInfo.getOutputType)
- assertEquals(
- classOf[KeyExtractingMapper[_, _]],
- keyExtractor.getUserCodeWrapper.getUserCodeClass)
- assertTrue(keyExtractor.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Test caused an error: " + e.getMessage)
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
deleted file mode 100644
index 9fb0980..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{GenericPairComparator, TypeComparator, TypeSerializer}
-import org.apache.flink.api.common.typeutils.base.{DoubleComparator, DoubleSerializer, IntComparator, IntSerializer}
-
-import org.apache.flink.api.java.typeutils.runtime.TupleComparator
-import org.apache.flink.api.scala.runtime.tuple.base.PairComparatorTestBase
-import org.apache.flink.api.scala.typeutils.CaseClassComparator
-
-class GenericPairComparatorTest
- extends PairComparatorTestBase[(Int, String, Double), (Int, Float, Long, Double)] {
-
- override protected def createComparator(ascending: Boolean):
- GenericPairComparator[(Int, String, Double), (Int, Float, Long, Double)] = {
- val fields1 = Array[Int](0, 2)
- val fields2 = Array[Int](0, 3)
-
- val comps1 =
- Array[TypeComparator[_]](new IntComparator(ascending), new DoubleComparator(ascending))
- val comps2 =
- Array[TypeComparator[_]](new IntComparator(ascending), new DoubleComparator(ascending))
-
- val sers1 =
- Array[TypeSerializer[_]](IntSerializer.INSTANCE, DoubleSerializer.INSTANCE)
- val sers2 =
- Array[TypeSerializer[_]](IntSerializer.INSTANCE, DoubleSerializer.INSTANCE)
-
- val comp1 = new CaseClassComparator[(Int, String, Double)](fields1, comps1, sers1)
- val comp2 = new CaseClassComparator[(Int, Float, Long, Double)](fields2, comps2, sers2)
-
- new GenericPairComparator[(Int, String, Double), (Int, Float, Long, Double)](comp1, comp2)
- }
-
- protected def getSortedTestData:
- (Array[(Int, String, Double)], Array[(Int, Float, Long, Double)]) = {
- (dataISD, dataIDL)
- }
-
- private val dataISD = Array(
- (4, "hello", 20.0),
- (4, "world", 23.2),
- (5, "hello", 18.0),
- (5, "world", 19.2),
- (6, "hello", 16.0),
- (6, "world", 17.2),
- (7, "hello", 14.0),
- (7,"world", 15.2))
-
- private val dataIDL = Array(
- (4, 0.11f, 14L, 20.0),
- (4, 0.221f, 15L, 23.2),
- (5, 0.33f, 15L, 18.0),
- (5, 0.44f, 20L, 19.2),
- (6, 0.55f, 20L, 16.0),
- (6, 0.66f, 29L, 17.2),
- (7, 0.77f, 29L, 14.0),
- (7, 0.88f, 34L, 15.2))
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
deleted file mode 100644
index adc46ac..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.TypeComparator
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-import org.apache.flink.api.scala._
-
-
-class TupleComparatorILD2Test extends TupleComparatorTestBase[(Int, Long, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(0, 1), Array(ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, Long, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, 14L, 20.0),
- (4, 15L, 23.2),
- (5, 15L, 20.0),
- (5, 20L, 20.0),
- (6, 20L, 23.2),
- (6, 29L, 20.0),
- (7, 29L, 20.0),
- (7, 34L, 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
deleted file mode 100644
index 377f985..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-
-class TupleComparatorILD3Test extends TupleComparatorTestBase[(Int, Long, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, Long, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, 4L, 20.0),
- (4, 4L, 23.2),
- (4, 9L, 20.0),
- (5, 4L, 20.0),
- (5, 4L, 23.2),
- (5, 9L, 20.0),
- (6, 4L, 20.0),
- (6, 4L, 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
deleted file mode 100644
index 1578951..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-
-class TupleComparatorILDC3Test extends TupleComparatorTestBase[(Int, Long, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(2, 0, 1), Array(ascending, ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, Long, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, 4L, 20.0),
- (5, 1L, 20.0),
- (5, 2L, 20.0),
- (5, 10L, 23.0),
- (5, 19L, 24.0),
- (5, 20L, 24.0),
- (5, 24L, 25.0),
- (5, 25L, 25.0)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
deleted file mode 100644
index 51e08cc..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-import org.apache.flink.api.scala._
-
-class TupleComparatorILDX1Test extends TupleComparatorTestBase[(Int, Long, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(1), Array(ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, Long, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, 4L, 20.0),
- (4, 5L, 23.2),
- (4, 9L, 20.0),
- (4, 10L, 24.0),
- (4, 19L, 23.2),
- (4, 20L, 24.0),
- (4, 24L, 20.0),
- (4, 25L, 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
deleted file mode 100644
index 3dbaabf..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-import org.apache.flink.api.scala._
-
-class TupleComparatorILDXC2Test extends TupleComparatorTestBase[(Int, Long, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(2, 1), Array(ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
- val ti = createTypeInformation[(Int, Long, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, Long, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, 4L, 20.0),
- (4, 5L, 20.0),
- (4, 3L, 23.0),
- (4, 19L, 23.0),
- (4, 17L, 24.0),
- (4, 18L, 24.0),
- (4, 24L, 25.0),
- (4, 25L, 25.0)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
deleted file mode 100644
index 252ae79..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-class TupleComparatorISD1Test extends TupleComparatorTestBase[(Int, String, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0), Array(ascending),0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, String, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, "hello", 20.0),
- (5, "hello", 23.2),
- (6, "world", 20.0),
- (7, "hello", 20.0),
- (8, "hello", 23.2),
- (9, "world", 20.0),
- (10, "hello", 20.0),
- (11, "hello", 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
deleted file mode 100644
index 37e775e..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-class TupleComparatorISD2Test extends TupleComparatorTestBase[(Int, String, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0, 1), Array(ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, String, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, "hello", 20.0),
- (4, "world", 23.2),
- (5, "hello", 20.0),
- (5, "world", 20.0),
- (6, "hello", 23.2),
- (6, "world", 20.0),
- (7, "hello", 20.0),
- (7, "world", 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
deleted file mode 100644
index 227b041..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
-
-class TupleComparatorISD3Test extends TupleComparatorTestBase[(Int, String, Double)] {
-
- protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
- }
-
- protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
- val ti = createTypeInformation[(Int, String, Double)]
- ti.createSerializer()
- }
-
- protected def getSortedTestData: Array[(Int, String, Double)] = {
- dataISD
- }
-
- private val dataISD = Array(
- (4, "hello", 20.0),
- (4, "hello", 23.2),
- (4, "world", 20.0),
- (5, "hello", 20.0),
- (5, "hello", 23.2),
- (5, "world", 20.0),
- (6, "hello", 20.0),
- (6, "hello", 23.2)
- )
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
deleted file mode 100644
index a4149dd..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.java.typeutils.runtime.AbstractGenericTypeSerializerTest._
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.util.StringUtils
-import org.junit.Assert
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-import scala.collection.JavaConverters._
-
-import java.util.Random
-
-class TupleSerializerTest {
-
- @Test
- def testTuple1Int(): Unit = {
- val testTuples = Array(Tuple1(42), Tuple1(1), Tuple1(0), Tuple1(-1), Tuple1(Int.MaxValue),
- Tuple1(Int.MinValue))
- runTests(testTuples)
- }
-
- @Test
- def testTuple1String(): Unit = {
- val rnd: Random = new Random(68761564135413L)
- val testTuples = Array(
- Tuple1(StringUtils.getRandomString(rnd, 10, 100)),
- Tuple1("abc"),
- Tuple1(""),
- Tuple1(StringUtils.getRandomString(rnd, 30, 170)),
- Tuple1(StringUtils.getRandomString(rnd, 15, 50)),
- Tuple1(""))
- runTests(testTuples)
- }
-
- @Test
- def testTuple1StringArray(): Unit = {
- val rnd: Random = new Random(289347567856686223L)
-
- val arr1 = Array(
- "abc",
- "",
- StringUtils.getRandomString(rnd, 10, 100),
- StringUtils.getRandomString(rnd, 15, 50),
- StringUtils.getRandomString(rnd, 30, 170),
- StringUtils.getRandomString(rnd, 14, 15),
- "")
- val arr2 = Array(
- "foo",
- "",
- StringUtils.getRandomString(rnd, 10, 100),
- StringUtils.getRandomString(rnd, 1000, 5000),
- StringUtils.getRandomString(rnd, 30000, 35000),
- StringUtils.getRandomString(rnd, 100 * 1024, 105 * 1024),
- "bar")
- val testTuples = Array(Tuple1(arr1), Tuple1(arr2))
- runTests(testTuples)
- }
-
- @Test
- def testTuple2StringDouble(): Unit = {
- val rnd: Random = new Random(807346528946L)
-
- val testTuples = Array(
- (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
- (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
- (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
- ("", rnd.nextDouble),
- (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
- (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble))
- runTests(testTuples)
- }
-
- @Test
- def testTuple2StringStringArray(): Unit = {
- val rnd: Random = new Random(289347567856686223L)
-
- val arr1 = Array(
- "abc",
- "",
- StringUtils.getRandomString(rnd, 10, 100),
- StringUtils.getRandomString(rnd, 15, 50),
- StringUtils.getRandomString(rnd, 30, 170),
- StringUtils.getRandomString(rnd, 14, 15), "")
- val arr2 = Array(
- "foo",
- "",
- StringUtils.getRandomString(rnd, 10, 100),
- StringUtils.getRandomString(rnd, 1000, 5000),
- StringUtils.getRandomString(rnd, 30000, 35000),
- StringUtils.getRandomString(rnd, 100 * 1024, 105 * 1024),
- "bar")
- val testTuples = Array(
- (StringUtils.getRandomString(rnd, 30, 170), arr1),
- (StringUtils.getRandomString(rnd, 30, 170), arr2),
- (StringUtils.getRandomString(rnd, 30, 170), arr1),
- (StringUtils.getRandomString(rnd, 30, 170), arr2),
- (StringUtils.getRandomString(rnd, 30, 170), arr2))
- runTests(testTuples)
- }
-
- @Test
- def testTuple5CustomObjects(): Unit = {
- val rnd: Random = new Random(807346528946L)
-
- val a = new SimpleTypes
- val b = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
- val c = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
- val d = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
- val e = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
- val f = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
- val g = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
- StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
-
- val o1 = new ComplexNestedObject1(5626435)
- val o2 = new ComplexNestedObject1(76923)
- val o3 = new ComplexNestedObject1(-1100)
- val o4 = new ComplexNestedObject1(0)
- val o5 = new ComplexNestedObject1(44)
-
- val co1 = new ComplexNestedObject2(rnd)
- val co2 = new ComplexNestedObject2
- val co3 = new ComplexNestedObject2(rnd)
- val co4 = new ComplexNestedObject2(rnd)
-
- val b1 = new Book(976243875L, "The Serialization Odysse", 42)
- val b2 = new Book(0L, "Debugging byte streams", 1337)
- val b3 = new Book(-1L, "Low level interfaces", 0xC0FFEE)
- val b4 = new Book(Long.MaxValue, "The joy of bits and bytes", 0xDEADBEEF)
- val b5 = new Book(Long.MaxValue, "Winnign a prize for creative test strings", 0xBADF00)
- val b6 = new Book(-2L, "Distributed Systems", 0xABCDEF0123456789L)
-
- val list = List("A", "B", "C", "D", "E")
- val ba1 = new BookAuthor(976243875L, list.asJava, "Arno Nym")
-
- val list2 = List[String]()
- val ba2 = new BookAuthor(987654321L, list2.asJava, "The Saurus")
-
- val testTuples = Array(
- (a, b1, o1, ba1, co1),
- (b, b2, o2, ba2, co2),
- (c, b3, o3, ba1, co3),
- (d, b2, o4, ba1, co4),
- (e, b4, o5, ba2, co4),
- (f, b5, o1, ba2, co4),
- (g, b6, o4, ba1, co2))
- runTests(testTuples)
- }
-
- private final def runTests[T <: Product : TypeInformation](instances: Array[T]) {
- try {
- val tupleTypeInfo = implicitly[TypeInformation[T]].asInstanceOf[TupleTypeInfoBase[T]]
- val serializer = tupleTypeInfo.createSerializer
- val tupleClass = tupleTypeInfo.getTypeClass
- val test = new TupleSerializerTestInstance[T](serializer, tupleClass, -1, instances)
- test.testAll()
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- Assert.fail(e.getMessage)
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
deleted file mode 100644
index 3b923bd..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime
-
-import org.junit.Assert._
-import org.apache.flink.api.common.typeutils.SerializerTestInstance
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.junit.Assert
-import org.junit.Test
-
-
-class TupleSerializerTestInstance[T <: Product] (
- serializer: TypeSerializer[T],
- typeClass: Class[T],
- length: Int,
- testData: Array[T])
- extends SerializerTestInstance[T](serializer, typeClass, length, testData: _*) {
-
- @Test
- override def testInstantiate(): Unit = {
- try {
- val serializer: TypeSerializer[T] = getSerializer
- val instance: T = serializer.createInstance
- assertNotNull("The created instance must not be null.", instance)
- val tpe: Class[T] = getTypeClass
- assertNotNull("The test is corrupt: type class is null.", tpe)
- // We cannot check this because Tuple1 instances are not actually of type Tuple1
- // but something like Tuple1$mcI$sp
-// assertEquals("Type of the instantiated object is wrong.", tpe, instance.getClass)
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-
- protected override def deepEquals(message: String, shouldTuple: T, isTuple: T) {
- Assert.assertEquals(shouldTuple.productArity, isTuple.productArity)
- for (i <- 0 until shouldTuple.productArity) {
- val should = shouldTuple.productElement(i)
- val is = isTuple.productElement(i)
- if (should.getClass.isArray) {
- should match {
- case booleans: Array[Boolean] =>
- Assert.assertTrue(message, booleans.sameElements(is.asInstanceOf[Array[Boolean]]))
- case bytes: Array[Byte] =>
- assertArrayEquals(message, bytes, is.asInstanceOf[Array[Byte]])
- case shorts: Array[Short] =>
- assertArrayEquals(message, shorts, is.asInstanceOf[Array[Short]])
- case ints: Array[Int] =>
- assertArrayEquals(message, ints, is.asInstanceOf[Array[Int]])
- case longs: Array[Long] =>
- assertArrayEquals(message, longs, is.asInstanceOf[Array[Long]])
- case floats: Array[Float] =>
- assertArrayEquals(message, floats, is.asInstanceOf[Array[Float]], 0.0f)
- case doubles: Array[Double] =>
- assertArrayEquals(message, doubles, is.asInstanceOf[Array[Double]], 0.0)
- case chars: Array[Char] =>
- assertArrayEquals(message, chars, is.asInstanceOf[Array[Char]])
- case _ =>
- assertArrayEquals(
- message,
- should.asInstanceOf[Array[AnyRef]],
- is.asInstanceOf[Array[AnyRef]])
- }
- } else {
- assertEquals(message, should, is)
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
deleted file mode 100644
index 9809dd2..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime.tuple.base
-
-import org.junit.Assert.assertTrue
-import org.junit.Assert.fail
-import org.apache.flink.api.common.typeutils.TypePairComparator
-import org.junit.Test
-
-/**
- * Abstract test base for PairComparators.
- */
-abstract class PairComparatorTestBase[T, R] {
- protected def createComparator(ascending: Boolean): TypePairComparator[T, R]
-
- protected def getSortedTestData: (Array[T], Array[R])
-
- @Test
- def testEqualityWithReference(): Unit = {
- try {
- val comparator = getComparator(true)
-
- val (dataT, dataR) = getSortedData
- for (i <- 0 until dataT.length) {
- comparator.setReference(dataT(i))
- assertTrue(comparator.equalToReference(dataR(i)))
- }
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-
- @Test
- def testInequalityWithReference(): Unit = {
- testGreatSmallAscDescWithReference(true)
- testGreatSmallAscDescWithReference(false)
- }
-
- protected def testGreatSmallAscDescWithReference(ascending: Boolean) {
- try {
- val (dataT, dataR) = getSortedData
- val comparator = getComparator(ascending)
- for (x <- 0 until (dataT.length - 1)) {
- for (y <- (x + 1) until dataR.length) {
- comparator.setReference(dataT(x))
- if (ascending) {
- assertTrue(comparator.compareToReference(dataR(y)) > 0)
- }
- else {
- assertTrue(comparator.compareToReference(dataR(y)) < 0)
- }
- }
- }
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-
- protected def getComparator(ascending: Boolean): TypePairComparator[T, R] = {
- val comparator: TypePairComparator[T, R] = createComparator(ascending)
- if (comparator == null) {
- throw new RuntimeException("Test case corrupt. Returns null as comparator.")
- }
- comparator
- }
-
- protected def getSortedData: (Array[T], Array[R]) = {
- val (dataT, dataR) = getSortedTestData
-
- if (dataT == null || dataR == null) {
- throw new RuntimeException("Test case corrupt. Returns null as test data.")
- }
- if (dataT.length < 2 || dataR.length < 2) {
- throw new RuntimeException("Test case does not provide enough sorted test data.")
- }
- (dataT, dataR)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
deleted file mode 100644
index 506b264..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.runtime.tuple.base
-
-import org.apache.flink.api.common.typeutils.ComparatorTestBase
-import org.apache.flink.api.scala.typeutils.{CaseClassSerializer, CaseClassComparator}
-import org.junit.Assert._
-
-abstract class TupleComparatorTestBase[T <: Product] extends ComparatorTestBase[T] {
- protected override def deepEquals(message: String, should: T, is: T) {
- for (i <- 0 until should.productArity) {
- assertEquals(should.productElement(i), is.productElement(i))
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
deleted file mode 100644
index 2b2d3a9..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.types
-
-import java.io.DataInput
-import java.io.DataOutput
-import org.apache.flink.api.common.typeinfo._
-import org.apache.flink.api.java.typeutils._
-import org.apache.flink.types.{IntValue, StringValue}
-import org.apache.hadoop.io.Writable
-import org.junit.Assert
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class MyWritable extends Writable {
- def write(out: DataOutput) {
- }
-
- def readFields(in: DataInput) {
- }
-}
-
-case class CustomCaseClass(a: String, b: Int)
-
-class CustomType(var myField1: String, var myField2: Int) {
- def this() {
- this(null, 0)
- }
-}
-
-class MyObject[A](var a: A) {
- def this() { this(null.asInstanceOf[A]) }
-}
-
-class TypeInformationGenTest {
-
- @Test
- def testBasicType(): Unit = {
- val ti = createTypeInformation[Boolean]
-
- Assert.assertTrue(ti.isBasicType)
- Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti)
- Assert.assertEquals(classOf[java.lang.Boolean], ti.getTypeClass)
- }
-
- @Test
- def testWritableType(): Unit = {
- val ti = createTypeInformation[MyWritable]
-
- Assert.assertTrue(ti.isInstanceOf[WritableTypeInfo[_]])
- Assert.assertEquals(classOf[MyWritable], ti.asInstanceOf[WritableTypeInfo[_]].getTypeClass)
- }
-
- @Test
- def testTupleWithBasicTypes(): Unit = {
- val ti = createTypeInformation[(Int, Long, Double, Float, Boolean, String, Char, Short, Byte)]
-
- Assert.assertTrue(ti.isTupleType)
- Assert.assertEquals(9, ti.getArity)
- Assert.assertTrue(ti.isInstanceOf[TupleTypeInfoBase[_]])
- val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
- Assert.assertEquals(classOf[Tuple9[_,_,_,_,_,_,_,_,_]], tti.getTypeClass)
- for (i <- 0 until 0) {
- Assert.assertTrue(tti.getTypeAt(i).isInstanceOf[BasicTypeInfo[_]])
- }
-
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, tti.getTypeAt(0))
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tti.getTypeAt(1))
- Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, tti.getTypeAt(2))
- Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, tti.getTypeAt(3))
- Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, tti.getTypeAt(4))
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(5))
- Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, tti.getTypeAt(6))
- Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, tti.getTypeAt(7))
- Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, tti.getTypeAt(8))
- }
-
- @Test
- def testTupleWithTuples(): Unit = {
- val ti = createTypeInformation[(Tuple1[String], Tuple1[Int], Tuple2[Long, Long])]
-
- Assert.assertTrue(ti.isTupleType())
- Assert.assertEquals(3, ti.getArity)
- Assert.assertTrue(ti.isInstanceOf[TupleTypeInfoBase[_]])
- val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
- Assert.assertEquals(classOf[Tuple3[_, _, _]], tti.getTypeClass)
- Assert.assertTrue(tti.getTypeAt(0).isTupleType())
- Assert.assertTrue(tti.getTypeAt(1).isTupleType())
- Assert.assertTrue(tti.getTypeAt(2).isTupleType())
- Assert.assertEquals(classOf[Tuple1[_]], tti.getTypeAt(0).getTypeClass)
- Assert.assertEquals(classOf[Tuple1[_]], tti.getTypeAt(1).getTypeClass)
- Assert.assertEquals(classOf[Tuple2[_, _]], tti.getTypeAt(2).getTypeClass)
- Assert.assertEquals(1, tti.getTypeAt(0).getArity)
- Assert.assertEquals(1, tti.getTypeAt(1).getArity)
- Assert.assertEquals(2, tti.getTypeAt(2).getArity)
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO,
- tti.getTypeAt(0).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO,
- tti.getTypeAt(1).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO,
- tti.getTypeAt(2).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO,
- tti.getTypeAt(2).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1))
- }
-
- @Test
- def testCaseClass(): Unit = {
- val ti = createTypeInformation[CustomCaseClass]
-
- Assert.assertTrue(ti.isTupleType)
- Assert.assertEquals(2, ti.getArity)
- Assert.assertEquals(
- BasicTypeInfo.STRING_TYPE_INFO,
- ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
- Assert.assertEquals(
- BasicTypeInfo.INT_TYPE_INFO,
- ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1))
- Assert.assertEquals(
- classOf[CustomCaseClass],ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeClass())
- }
-
- @Test
- def testCustomType(): Unit = {
- val ti = createTypeInformation[CustomType]
-
- Assert.assertFalse(ti.isBasicType)
- Assert.assertFalse(ti.isTupleType)
- Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
- Assert.assertEquals(ti.getTypeClass, classOf[CustomType])
- }
-
- @Test
- def testTupleWithCustomType(): Unit = {
- val ti = createTypeInformation[(Long, CustomType)]
-
- Assert.assertTrue(ti.isTupleType)
- Assert.assertEquals(2, ti.getArity)
- val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
- Assert.assertEquals(classOf[Tuple2[_, _]], tti.getTypeClass)
- Assert.assertEquals(classOf[java.lang.Long], tti.getTypeAt(0).getTypeClass)
- Assert.assertTrue(tti.getTypeAt(1).isInstanceOf[PojoTypeInfo[_]])
- Assert.assertEquals(classOf[CustomType], tti.getTypeAt(1).getTypeClass)
- }
-
- @Test
- def testValue(): Unit = {
- val ti = createTypeInformation[StringValue]
-
- Assert.assertFalse(ti.isBasicType)
- Assert.assertFalse(ti.isTupleType)
- Assert.assertTrue(ti.isInstanceOf[ValueTypeInfo[_]])
- Assert.assertEquals(ti.getTypeClass, classOf[StringValue])
- Assert.assertTrue(TypeExtractor.getForClass(classOf[StringValue])
- .isInstanceOf[ValueTypeInfo[_]])
- Assert.assertEquals(TypeExtractor.getForClass(classOf[StringValue]).getTypeClass,
- ti.getTypeClass)
- }
-
- @Test
- def testTupleOfValues(): Unit = {
- val ti = createTypeInformation[(StringValue, IntValue)]
- Assert.assertFalse(ti.isBasicType)
- Assert.assertTrue(ti.isTupleType)
- Assert.assertEquals(
- classOf[StringValue],
- ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0).getTypeClass)
- Assert.assertEquals(
- classOf[IntValue],
- ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1).getTypeClass)
- }
-
-
- @Test
- def testBasicArray(): Unit = {
- val ti = createTypeInformation[Array[String]]
-
- Assert.assertFalse(ti.isBasicType)
- Assert.assertFalse(ti.isTupleType)
- Assert.assertTrue(ti.isInstanceOf[BasicArrayTypeInfo[_, _]] ||
- ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
- if (ti.isInstanceOf[BasicArrayTypeInfo[_, _]]) {
- Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, ti)
- }
- else {
- Assert.assertEquals(
- BasicTypeInfo.STRING_TYPE_INFO,
- ti.asInstanceOf[ObjectArrayTypeInfo[_, _]].getComponentInfo)
- }
- }
-
- @Test
- def testPrimitiveArray(): Unit = {
- val ti = createTypeInformation[Array[Boolean]]
-
- Assert.assertTrue(ti.isInstanceOf[PrimitiveArrayTypeInfo[_]])
- Assert.assertEquals(ti, PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO)
- }
-
- @Test
- def testCustomArray(): Unit = {
- val ti = createTypeInformation[Array[CustomType]]
- Assert.assertTrue(ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
- Assert.assertEquals(
- classOf[CustomType],
- ti.asInstanceOf[ObjectArrayTypeInfo[_, _]].getComponentType)
- }
-
- @Test
- def testTupleArray(): Unit = {
- val ti = createTypeInformation[Array[(String, String)]]
-
- Assert.assertTrue(ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
- val oati = ti.asInstanceOf[ObjectArrayTypeInfo[_, _]]
- Assert.assertTrue(oati.getComponentInfo.isTupleType)
- val tti = oati.getComponentInfo.asInstanceOf[TupleTypeInfoBase[_]]
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0))
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1))
- }
-
- @Test
- def testParamertizedCustomObject(): Unit = {
- val ti = createTypeInformation[MyObject[String]]
-
- Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
- }
-
- @Test
- def testTupleWithPrimitiveArray(): Unit = {
- val ti = createTypeInformation[(Array[Int], Array[Double], Array[Long],
- Array[Byte], Array[Char], Array[Float], Array[Short], Array[Boolean],
- Array[String])]
-
- val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
- Assert.assertEquals(PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(0))
- Assert.assertEquals(PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(1))
- Assert.assertEquals(PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(2))
- Assert.assertEquals(PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(3))
- Assert.assertEquals(PrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(4))
- Assert.assertEquals(PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(5))
- Assert.assertEquals(PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(6))
- Assert.assertEquals(PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(7))
- Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, tti.getTypeAt(8))
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
deleted file mode 100644
index 09e049b..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.util
-
-import org.apache.hadoop.io.IntWritable
-
-import org.apache.flink.api.scala._
-
-import scala.collection.mutable
-import scala.reflect.classTag
-import scala.util.Random
-
-/**
- * #################################################################################################
- *
- * BE AWARE THAT OTHER TESTS DEPEND ON THIS TEST DATA.
- * IF YOU MODIFY THE DATA MAKE SURE YOU CHECK THAT ALL TESTS ARE STILL WORKING!
- *
- * #################################################################################################
- */
-object CollectionDataSets {
- def get3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
- val data = new mutable.MutableList[(Int, Long, String)]
- data.+=((1, 1L, "Hi"))
- data.+=((2, 2L, "Hello"))
- data.+=((3, 2L, "Hello world"))
- data.+=((4, 3L, "Hello world, how are you?"))
- data.+=((5, 3L, "I am fine."))
- data.+=((6, 3L, "Luke Skywalker"))
- data.+=((7, 4L, "Comment#1"))
- data.+=((8, 4L, "Comment#2"))
- data.+=((9, 4L, "Comment#3"))
- data.+=((10, 4L, "Comment#4"))
- data.+=((11, 5L, "Comment#5"))
- data.+=((12, 5L, "Comment#6"))
- data.+=((13, 5L, "Comment#7"))
- data.+=((14, 5L, "Comment#8"))
- data.+=((15, 5L, "Comment#9"))
- data.+=((16, 6L, "Comment#10"))
- data.+=((17, 6L, "Comment#11"))
- data.+=((18, 6L, "Comment#12"))
- data.+=((19, 6L, "Comment#13"))
- data.+=((20, 6L, "Comment#14"))
- data.+=((21, 6L, "Comment#15"))
- Random.shuffle(data)
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmall3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
- val data = new mutable.MutableList[(Int, Long, String)]
- data.+=((1, 1L, "Hi"))
- data.+=((2, 2L, "Hello"))
- data.+=((3, 2L, "Hello world"))
- env.fromCollection(Random.shuffle(data))
- }
-
- def get5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
- val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
- data.+=((1, 1L, 0, "Hallo", 1L))
- data.+=((2, 2L, 1, "Hallo Welt", 2L))
- data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
- data.+=((3, 4L, 3, "Hallo Welt wie gehts?", 2L))
- data.+=((3, 5L, 4, "ABC", 2L))
- data.+=((3, 6L, 5, "BCD", 3L))
- data.+=((4, 7L, 6, "CDE", 2L))
- data.+=((4, 8L, 7, "DEF", 1L))
- data.+=((4, 9L, 8, "EFG", 1L))
- data.+=((4, 10L, 9, "FGH", 2L))
- data.+=((5, 11L, 10, "GHI", 1L))
- data.+=((5, 12L, 11, "HIJ", 3L))
- data.+=((5, 13L, 12, "IJK", 3L))
- data.+=((5, 14L, 13, "JKL", 2L))
- data.+=((5, 15L, 14, "KLM", 2L))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmall5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
- val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
- data.+=((1, 1L, 0, "Hallo", 1L))
- data.+=((2, 2L, 1, "Hallo Welt", 2L))
- data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmallNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
- val data = new mutable.MutableList[((Int, Int), String)]
- data.+=(((1, 1), "one"))
- data.+=(((2, 2), "two"))
- data.+=(((3, 3), "three"))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getGroupSortedNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
- val data = new mutable.MutableList[((Int, Int), String)]
- data.+=(((1, 3), "a"))
- data.+=(((1, 2), "a"))
- data.+=(((2, 1), "a"))
- data.+=(((2, 2), "b"))
- data.+=(((3, 3), "c"))
- data.+=(((3, 6), "c"))
- data.+=(((4, 9), "c"))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getStringDataSet(env: ExecutionEnvironment): DataSet[String] = {
- val data = new mutable.MutableList[String]
- data.+=("Hi")
- data.+=("Hello")
- data.+=("Hello world")
- data.+=("Hello world, how are you?")
- data.+=("I am fine.")
- data.+=("Luke Skywalker")
- data.+=("Random comment")
- data.+=("LOL")
- env.fromCollection(Random.shuffle(data))
- }
-
- def getIntDataSet(env: ExecutionEnvironment): DataSet[Int] = {
- val data = new mutable.MutableList[Int]
- data.+=(1)
- data.+=(2)
- data.+=(2)
- data.+=(3)
- data.+=(3)
- data.+=(3)
- data.+=(4)
- data.+=(4)
- data.+=(4)
- data.+=(4)
- data.+=(5)
- data.+=(5)
- data.+=(5)
- data.+=(5)
- data.+=(5)
- env.fromCollection(Random.shuffle(data))
- }
-
- def getCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
- val data = new mutable.MutableList[CustomType]
- data.+=(new CustomType(1, 0L, "Hi"))
- data.+=(new CustomType(2, 1L, "Hello"))
- data.+=(new CustomType(2, 2L, "Hello world"))
- data.+=(new CustomType(3, 3L, "Hello world, how are you?"))
- data.+=(new CustomType(3, 4L, "I am fine."))
- data.+=(new CustomType(3, 5L, "Luke Skywalker"))
- data.+=(new CustomType(4, 6L, "Comment#1"))
- data.+=(new CustomType(4, 7L, "Comment#2"))
- data.+=(new CustomType(4, 8L, "Comment#3"))
- data.+=(new CustomType(4, 9L, "Comment#4"))
- data.+=(new CustomType(5, 10L, "Comment#5"))
- data.+=(new CustomType(5, 11L, "Comment#6"))
- data.+=(new CustomType(5, 12L, "Comment#7"))
- data.+=(new CustomType(5, 13L, "Comment#8"))
- data.+=(new CustomType(5, 14L, "Comment#9"))
- data.+=(new CustomType(6, 15L, "Comment#10"))
- data.+=(new CustomType(6, 16L, "Comment#11"))
- data.+=(new CustomType(6, 17L, "Comment#12"))
- data.+=(new CustomType(6, 18L, "Comment#13"))
- data.+=(new CustomType(6, 19L, "Comment#14"))
- data.+=(new CustomType(6, 20L, "Comment#15"))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmallCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
- val data = new mutable.MutableList[CustomType]
- data.+=(new CustomType(1, 0L, "Hi"))
- data.+=(new CustomType(2, 1L, "Hello"))
- data.+=(new CustomType(2, 2L, "Hello world"))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmallTuplebasedPojoMatchingDataSet(env: ExecutionEnvironment):
- DataSet[(Int, String, Int, Int, Long, String, Long)] = {
- val data = new mutable.MutableList[(Int, String, Int, Int, Long, String, Long)]
- data.+=((1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=((2, "Second", 20, 200, 2000L, "Two", 20000L))
- data.+=((3, "Third", 30, 300, 3000L, "Three", 30000L))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getSmallPojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
- val data = new mutable.MutableList[POJO]
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
- data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
- env.fromCollection(Random.shuffle(data))
- }
-
- def getDuplicatePojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
- val data = new mutable.MutableList[POJO]
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
- data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
- data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
- data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
- env.fromCollection(data)
- }
-
- def getCrazyNestedDataSet(env: ExecutionEnvironment): DataSet[CrazyNested] = {
- val data = new mutable.MutableList[CrazyNested]
- data.+=(new CrazyNested("aa"))
- data.+=(new CrazyNested("bb"))
- data.+=(new CrazyNested("bb"))
- data.+=(new CrazyNested("cc"))
- data.+=(new CrazyNested("cc"))
- data.+=(new CrazyNested("cc"))
- env.fromCollection(data)
- }
-
- def getPojoContainingTupleAndWritable(env: ExecutionEnvironment): DataSet[CollectionDataSets
- .PojoContainingTupleAndWritable] = {
- val data = new
- mutable.MutableList[PojoContainingTupleAndWritable]
- data.+=(new PojoContainingTupleAndWritable(1, 10L, 100L))
- data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
- data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
- data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
- data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
- data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
- env.fromCollection(data)
- }
-
- def getGroupSortedPojoContainingTupleAndWritable(env: ExecutionEnvironment):
- DataSet[PojoContainingTupleAndWritable] = {
- val data = new mutable.MutableList[PojoContainingTupleAndWritable]
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(1, 10L, 100L))
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 20L, 200L))
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 20L, 201L))
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 200L))
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 600L))
- data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 400L))
- env.fromCollection(data)
- }
-
- def getTupleContainingPojos(env: ExecutionEnvironment): DataSet[(Int, CrazyNested, POJO)] = {
- val data = new mutable.MutableList[(Int, CrazyNested, POJO)]
- data.+=((
- 1,
- new CrazyNested("one", "uno", 1L),
- new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
- data.+=((
- 1,
- new CrazyNested("one", "uno", 1L),
- new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
- data.+=((
- 1,
- new CrazyNested("one", "uno", 1L),
- new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
- data.+=((
- 2,
- new CrazyNested("two", "duo", 2L),
- new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
- env.fromCollection(data)
- }
-
- def getSmallTuplebasedDataSetMatchingPojo(env: ExecutionEnvironment):
- DataSet[(Long, Integer, Integer, Long, String, Integer, String)] = {
- val data = new mutable.MutableList[(Long, Integer, Integer, Long, String, Integer, String)]
- data.+=((10000L, 10, 100, 1000L, "One", 1, "First"))
- data.+=((20000L, 20, 200, 2000L, "Two", 2, "Second"))
- data.+=((30000L, 30, 300, 3000L, "Three", 3, "Third"))
- env.fromCollection(data)
- }
-
- def getPojoWithMultiplePojos(env: ExecutionEnvironment): DataSet[CollectionDataSets
- .PojoWithMultiplePojos] = {
- val data = new mutable.MutableList[CollectionDataSets
- .PojoWithMultiplePojos]
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("a", "aa", "b", "bb", 1))
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("d", "dd", "e", "ee", 3))
- data.+=(new CollectionDataSets.PojoWithMultiplePojos("d", "dd", "e", "ee", 3))
- env.fromCollection(data)
- }
-
- case class MutableTuple3[T1, T2, T3](var _1: T1, var _2: T2, var _3: T3)
-
- class CustomType(var myInt: Int, var myLong: Long, var myString: String) {
- def this() {
- this(0, 0, "")
- }
-
- override def toString: String = {
- myInt + "," + myLong + "," + myString
- }
- }
-
- class POJO(
- var number: Int,
- var str: String,
- var nestedTupleWithCustom: (Int, CustomType),
- var nestedPojo: NestedPojo) {
- def this() {
- this(0, "", null, null)
- }
-
- def this(i0: Int, s0: String, i1: Int, i2: Int, l0: Long, s1: String, l1: Long) {
- this(i0, s0, (i1, new CustomType(i2, l0, s1)), new NestedPojo(l1))
- }
-
- override def toString: String = {
- number + " " + str + " " + nestedTupleWithCustom + " " + nestedPojo.longNumber
- }
-
- @transient var ignoreMe: Long = 1L
- }
-
- class NestedPojo(var longNumber: Long) {
- def this() {
- this(0)
- }
- }
-
- class CrazyNested(var nest_Lvl1: CrazyNestedL1, var something: Long) {
- def this() {
- this(new CrazyNestedL1, 0)
- }
-
- def this(set: String) {
- this()
- nest_Lvl1 = new CrazyNestedL1
- nest_Lvl1.nest_Lvl2 = new CrazyNestedL2
- nest_Lvl1.nest_Lvl2.nest_Lvl3 = new CrazyNestedL3
- nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4 = new CrazyNestedL4
- nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal = set
- }
-
- def this(set: String, second: String, s: Long) {
- this(set)
- something = s
- nest_Lvl1.a = second
- }
- }
-
- class CrazyNestedL1 {
- var a: String = null
- var b: Int = 0
- var nest_Lvl2: CrazyNestedL2 = null
- }
-
- class CrazyNestedL2 {
- var nest_Lvl3: CrazyNestedL3 = null
- }
-
- class CrazyNestedL3 {
- var nest_Lvl4: CrazyNestedL4 = null
- }
-
- class CrazyNestedL4 {
- var f1nal: String = null
- }
-
- class PojoContainingTupleAndWritable(
- var someInt: Int,
- var someString: String,
- var hadoopFan: IntWritable,
- var theTuple: (Long, Long)) {
- def this() {
- this(0, "", new IntWritable(0), (0, 0))
- }
-
- def this(i: Int, l1: Long, l2: Long) {
- this()
- hadoopFan = new IntWritable(i)
- someInt = i
- theTuple = (l1, l2)
- }
-
- }
-
- class Pojo1 {
- var a: String = null
- var b: String = null
-
- override def toString = s"Pojo1 a=$a b=$b"
- }
-
- class Pojo2 {
- var a2: String = null
- var b2: String = null
-
- override def toString = s"Pojo2 a2=$a2 b2=$b2"
- }
-
- class PojoWithMultiplePojos {
-
- def this(a: String, b: String, a1: String, b1: String, i0: Int) {
- this()
- p1 = new Pojo1
- p1.a = a
- p1.b = b
- p2 = new Pojo2
- p2.a2 = a1
- p2.b2 = b1
- this.i0 = i0
- }
-
- var p1: Pojo1 = null
- var p2: Pojo2 = null
- var i0: Int = 0
-
- override def toString = s"PojoWithMultiplePojos p1=$p1 p2=$p2 i0=$i0"
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/pom.xml
----------------------------------------------------------------------
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index 16dccb6..ff2b607 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -36,54 +36,63 @@ under the License.
<packaging>jar</packaging>
<dependencies>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-compiler</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java-examples</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala-examples</artifactId>
@@ -91,17 +100,162 @@ under the License.
<scope>test</scope>
</dependency>
- <!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
- <scope>provided</scope>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_2.10</artifactId>
+ <version>2.2.0</version>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
+
+ <!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) test-compile phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../tools/maven/scalastyle-config.xml</configLocation>
+ <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
@@ -113,6 +267,7 @@ under the License.
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -125,6 +280,7 @@ under the License.
</excludes>
</configuration>
</plugin>
+
<plugin>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
new file mode 100644
index 0000000..729fda4
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala
+
+import org.junit.Test
+import org.apache.flink.api.common.InvalidProgramException
+
+// Verify that the sanity checking in delta iterations works. We just
+// have a dummy job that is not meant to be executed. Only verify that
+// the join/coGroup inside the iteration is checked.
+class DeltaIterationSanityCheckTest extends Serializable {
+
+ @Test
+ def testCorrectJoinWithSolution1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = s.join(ws).where("_1").equalTo("_1") { (l, r) => l }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test
+ def testCorrectJoinWithSolution2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = ws.join(s).where("_1").equalTo("_1") { (l, r) => l }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectJoinWithSolution1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = s.join(ws).where("_2").equalTo("_2") { (l, r) => l }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectJoinWithSolution2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = ws.join(s).where("_2").equalTo("_2") { (l, r) => l }
+ (result, ws)
+ }
+
+ iteration.print() }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectJoinWithSolution3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_2")) { (s, ws) =>
+ val result = ws.join(s).where("_1").equalTo("_1") { (l, r) => l }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test
+ def testCorrectCoGroupWithSolution1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = s.coGroup(ws).where("_1").equalTo("_1") { (l, r) => l.min }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test
+ def testCorrectCoGroupWithSolution2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = ws.coGroup(s).where("_1").equalTo("_1") { (l, r) => l.min }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectCoGroupWithSolution1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = s.coGroup(ws).where("_2").equalTo("_2") { (l, r) => l.min }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectCoGroupWithSolution2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
+ val result = ws.coGroup(s).where("_2").equalTo("_2") { (l, r) => l.min }
+ (result, ws)
+ }
+
+ iteration.print() }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testIncorrectCoGroupWithSolution3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val solutionInput = env.fromElements((1, "1"))
+ val worksetInput = env.fromElements((2, "2"))
+
+ val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_2")) { (s, ws) =>
+ val result = ws.coGroup(s).where("_1").equalTo("_1") { (l, r) => l.min }
+ (result, ws)
+ }
+
+ iteration.print()
+ }
+}
[3/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
new file mode 100644
index 0000000..0219154
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.junit.Assert
+import org.apache.flink.api.common.InvalidProgramException
+import org.junit.Ignore
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class JoinOperatorTest {
+
+ private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+ private val customTypeData = Array[CustomType](new CustomType())
+ private val emptyLongData = Array[Long]()
+
+ @Test
+ def testJoinKeyIndices1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ ds1.join(ds2).where(0).equalTo(0)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyIndices2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, incompatible key types
+ ds1.join(ds2).where(0).equalTo(2)
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyIndices3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, non-matching number of key indices
+ ds1.join(ds2).where(0, 1).equalTo(2)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testJoinKeyIndices4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, index out of range
+ ds1.join(ds2).where(5).equalTo(0)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testJoinKeyIndices5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, negative position
+ ds1.join(ds2).where(-1).equalTo(-1)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testJoinKeyIndices6(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, key index on custom type
+ ds1.join(ds2).where(5).equalTo(0)
+ }
+
+ @Test
+ def testJoinKeyFields1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ ds1.join(ds2).where("_1").equalTo("_1")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyFields2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, incompatible field types
+ ds1.join(ds2).where("_1").equalTo("_3")
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyFields3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, non-matching number of key indices
+
+ ds1.join(ds2).where("_1", "_2").equalTo("_3")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testJoinKeyFields4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, non-existent key
+ ds1.join(ds2).where("foo").equalTo("_1")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testJoinKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should not work, non-matching number of key indices
+ ds1.join(ds2).where("_1").equalTo("bar")
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testJoinKeyFields6(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, field key on custom type
+ ds1.join(ds2).where("_2").equalTo("_1")
+ }
+
+ @Test
+ def testJoinKeyExpressions1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should work
+ try {
+ ds1.join(ds2).where("myInt").equalTo("myInt")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyExpressions2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, incompatible join key types
+ ds1.join(ds2).where("myInt").equalTo("myString")
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyExpressions3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, incompatible number of keys
+ ds1.join(ds2).where("myInt", "myString").equalTo("myInt")
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testJoinKeyExpressions4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, join key non-existent
+ ds1.join(ds2).where("myNonExistent").equalTo("i")
+ }
+
+ @Test
+ def testJoinKeySelectors1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should work
+ try {
+ ds1.join(ds2).where { _.myLong} equalTo { _.myLong }
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testJoinKeyMixing1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(customTypeData)
+ val ds2 = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ ds1.join(ds2).where { _.myLong }.equalTo(3)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testJoinKeyMixing2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should work
+ try {
+ ds1.join(ds2).where(3).equalTo { _.myLong }
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyMixing3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, incompatible types
+ ds1.join(ds2).where(2).equalTo { _.myLong }
+ }
+
+ @Test(expected = classOf[IncompatibleKeysException])
+ def testJoinKeyMixing4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = env.fromCollection(emptyTupleData)
+ val ds2 = env.fromCollection(customTypeData)
+
+ // should not work, more than one field position key
+ ds1.join(ds2).where(1, 3) equalTo { _.myLong }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
new file mode 100644
index 0000000..71c6b8b
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/MapITCase.scala
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichMapFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.Assert
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object MapProgs {
+ var NUM_PROGRAMS: Int = 9
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Test identity map with basic type
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getStringDataSet(env)
+ val identityMapDs = ds.map( t => t)
+ identityMapDs.writeAsText(resultPath)
+ env.execute()
+ "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
+ ".\n" + "Luke Skywalker\n" + "Random comment\n" + "LOL\n"
+
+ case 2 =>
+ /*
+ * Test identity map with a tuple
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val identityMapDs = ds.map( t => t )
+ identityMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
+ "how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" + "7,4," +
+ "Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" + "11,5," +
+ "Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
+ "Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
+ "6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
+
+ case 3 =>
+ /*
+ * Test type conversion mapper (Custom -> Tuple)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val typeConversionMapDs = ds.map( c => (c.myInt, c.myLong, c.myString) )
+ typeConversionMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,0,Hi\n" + "2,1,Hello\n" + "2,2,Hello world\n" + "3,3,Hello world, " +
+ "how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n" + "4,6," +
+ "Comment#1\n" + "4,7,Comment#2\n" + "4,8,Comment#3\n" + "4,9,Comment#4\n" + "5,10," +
+ "Comment#5\n" + "5,11,Comment#6\n" + "5,12,Comment#7\n" + "5,13,Comment#8\n" + "5,14," +
+ "Comment#9\n" + "6,15,Comment#10\n" + "6,16,Comment#11\n" + "6,17,Comment#12\n" + "6," +
+ "18,Comment#13\n" + "6,19,Comment#14\n" + "6,20,Comment#15\n"
+
+ case 4 =>
+ /*
+ * Test type conversion mapper (Tuple -> Basic)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val typeConversionMapDs = ds.map(_._3)
+ typeConversionMapDs.writeAsText(resultPath)
+ env.execute()
+ "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
+ ".\n" + "Luke Skywalker\n" + "Comment#1\n" + "Comment#2\n" + "Comment#3\n" +
+ "Comment#4\n" + "Comment#5\n" + "Comment#6\n" + "Comment#7\n" + "Comment#8\n" +
+ "Comment#9\n" + "Comment#10\n" + "Comment#11\n" + "Comment#12\n" + "Comment#13\n" +
+ "Comment#14\n" + "Comment#15\n"
+
+ case 5 =>
+ /*
+ * Test mapper on tuple - Increment Integer field, reorder second and third fields
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val tupleMapDs = ds.map( t => (t._1 + 1, t._3, t._2) )
+ tupleMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "2,Hi,1\n" + "3,Hello,2\n" + "4,Hello world,2\n" + "5,Hello world, how are you?," +
+ "3\n" + "6,I am fine.,3\n" + "7,Luke Skywalker,3\n" + "8,Comment#1,4\n" + "9,Comment#2," +
+ "4\n" + "10,Comment#3,4\n" + "11,Comment#4,4\n" + "12,Comment#5,5\n" + "13,Comment#6," +
+ "5\n" + "14,Comment#7,5\n" + "15,Comment#8,5\n" + "16,Comment#9,5\n" + "17,Comment#10," +
+ "6\n" + "18,Comment#11,6\n" + "19,Comment#12,6\n" + "20,Comment#13,6\n" + "21," +
+ "Comment#14,6\n" + "22,Comment#15,6\n"
+
+ case 6 =>
+ /*
+ * Test mapper on Custom - lowercase myString
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val customMapDs = ds.map { c => c.myString = c.myString.toLowerCase; c }
+ customMapDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,hi\n" + "2,1,hello\n" + "2,2,hello world\n" + "3,3,hello world, " +
+ "how are you?\n" + "3,4,i am fine.\n" + "3,5,luke skywalker\n" + "4,6," +
+ "comment#1\n" + "4,7,comment#2\n" + "4,8,comment#3\n" + "4,9,comment#4\n" + "5,10," +
+ "comment#5\n" + "5,11,comment#6\n" + "5,12,comment#7\n" + "5,13,comment#8\n" + "5,14," +
+ "comment#9\n" + "6,15,comment#10\n" + "6,16,comment#11\n" + "6,17,comment#12\n" + "6," +
+ "18,comment#13\n" + "6,19,comment#14\n" + "6,20,comment#15\n"
+
+ case 7 =>
+ /*
+ * Test mapper if UDF returns input object - increment first field of a tuple
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env).map {
+ t => MutableTuple3(t._1, t._2, t._3)
+ }
+ val inputObjMapDs = ds.map { t => t._1 = t._1 + 1; t }
+ inputObjMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "2,1,Hi\n" + "3,2,Hello\n" + "4,2,Hello world\n" + "5,3,Hello world, " +
+ "how are you?\n" + "6,3,I am fine.\n" + "7,3,Luke Skywalker\n" + "8,4," +
+ "Comment#1\n" + "9,4,Comment#2\n" + "10,4,Comment#3\n" + "11,4,Comment#4\n" + "12,5," +
+ "Comment#5\n" + "13,5,Comment#6\n" + "14,5,Comment#7\n" + "15,5,Comment#8\n" + "16,5," +
+ "Comment#9\n" + "17,6,Comment#10\n" + "18,6,Comment#11\n" + "19,6,Comment#12\n" + "20," +
+ "6,Comment#13\n" + "21,6,Comment#14\n" + "22,6,Comment#15\n"
+
+ case 8 =>
+ /*
+ * Test map with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ints = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val bcMapDs = ds.map(
+ new RichMapFunction[(Int, Long, String), (Int, Long, String)] {
+ var f2Replace = 0
+ override def open(config: Configuration): Unit = {
+ val ints = getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ f2Replace = ints.sum
+ }
+ override def map(in: (Int, Long, String)): (Int, Long, String) = {
+ in.copy(_1 = f2Replace)
+ }
+ }).withBroadcastSet(ints, "ints")
+ bcMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "55,1,Hi\n" + "55,2,Hello\n" + "55,2,Hello world\n" + "55,3,Hello world, " +
+ "how are you?\n" + "55,3,I am fine.\n" + "55,3,Luke Skywalker\n" + "55,4," +
+ "Comment#1\n" + "55,4,Comment#2\n" + "55,4,Comment#3\n" + "55,4,Comment#4\n" + "55,5," +
+ "Comment#5\n" + "55,5,Comment#6\n" + "55,5,Comment#7\n" + "55,5,Comment#8\n" + "55,5," +
+ "Comment#9\n" + "55,6,Comment#10\n" + "55,6,Comment#11\n" + "55,6,Comment#12\n" + "55," +
+ "6,Comment#13\n" + "55,6,Comment#14\n" + "55,6,Comment#15\n"
+
+ case 9 =>
+ /*
+ * Test passing configuration object.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+ val conf = new Configuration
+ val testKey = "testVariable"
+ val testValue = 666
+ conf.setInteger(testKey, testValue)
+ val bcMapDs = ds.map(
+ new RichMapFunction[(Int, Long, String), (Int, Long, String)] {
+ override def open(config: Configuration): Unit = {
+ val fromConfig = config.getInteger(testKey, -1)
+ Assert.assertEquals(testValue, fromConfig)
+ }
+ override def map(in: (Int, Long, String)): (Int, Long, String) = {
+ in
+ }
+ }).withParameters(conf)
+ bcMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class MapITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = MapProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object MapITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to MapProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
new file mode 100644
index 0000000..07d4897
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.{RichFilterFunction, RichMapFunction}
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object PartitionProgs {
+ var NUM_PROGRAMS: Int = 7
+
+ def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Test hash partition by tuple field
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val unique = ds.partitionByHash(1).mapPartition( _.map(_._2).toSet )
+
+ unique.writeAsText(resultPath)
+ env.execute()
+
+ "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
+
+ case 2 =>
+ /*
+ * Test hash partition by key selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val unique = ds.partitionByHash( _._2 ).mapPartition( _.map(_._2).toSet )
+
+ unique.writeAsText(resultPath)
+ env.execute()
+ "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
+
+ case 3 =>
+ /*
+ * Test forced rebalancing
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.generateSequence(1, 3000)
+
+ val skewed = ds.filter(_ > 780)
+ val rebalanced = skewed.rebalance()
+
+ val countsInPartition = rebalanced.map( new RichMapFunction[Long, (Int, Long)] {
+ def map(in: Long) = {
+ (getRuntimeContext.getIndexOfThisSubtask, 1)
+ }
+ })
+ .groupBy(0)
+ .reduce { (v1, v2) => (v1._1, v1._2 + v2._2) }
+ // round counts to mitigate runtime scheduling effects (lazy split assignment)
+ .map { in => (in._1, in._2 / 10) }
+
+ countsInPartition.writeAsText(resultPath)
+ env.execute()
+
+ val numPerPartition : Int = 2220 / env.getDegreeOfParallelism / 10
+ var result = ""
+ for (i <- 0 until env.getDegreeOfParallelism) {
+ result += "(" + i + "," + numPerPartition + ")\n"
+ }
+ result
+
+ case 4 =>
+ // Verify that mapPartition operation after repartition picks up correct
+ // DOP
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ env.setDegreeOfParallelism(1)
+
+ val unique = ds.partitionByHash(1)
+ .setParallelism(4)
+ .mapPartition( _.map(_._2).toSet )
+
+ unique.writeAsText(resultPath)
+ env.execute()
+
+ "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
+
+ case 5 =>
+ // Verify that map operation after repartition picks up correct
+ // DOP
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ env.setDegreeOfParallelism(1)
+
+ val count = ds.partitionByHash(0).setParallelism(4).map(
+ new RichMapFunction[(Int, Long, String), Tuple1[Int]] {
+ var first = true
+ override def map(in: (Int, Long, String)): Tuple1[Int] = {
+ // only output one value with count 1
+ if (first) {
+ first = false
+ Tuple1(1)
+ } else {
+ Tuple1(0)
+ }
+ }
+ }).sum(0)
+
+ count.writeAsText(resultPath)
+ env.execute()
+
+ if (onCollection) "(1)\n" else "(4)\n"
+
+ case 6 =>
+ // Verify that filter operation after repartition picks up correct
+ // DOP
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ env.setDegreeOfParallelism(1)
+
+ val count = ds.partitionByHash(0).setParallelism(4).filter(
+ new RichFilterFunction[(Int, Long, String)] {
+ var first = true
+ override def filter(in: (Int, Long, String)): Boolean = {
+ // only output one value with count 1
+ if (first) {
+ first = false
+ true
+ } else {
+ false
+ }
+ }
+ })
+ .map( _ => Tuple1(1)).sum(0)
+
+ count.writeAsText(resultPath)
+ env.execute()
+
+ if (onCollection) "(1)\n" else "(4)\n"
+
+ case 7 =>
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(3)
+ val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
+ val uniqLongs = ds
+ .partitionByHash("nestedPojo.longNumber")
+ .setParallelism(4)
+ .mapPartition( _.map(_.nestedPojo.longNumber).toSet )
+
+ uniqLongs.writeAsText(resultPath)
+ env.execute()
+ "10000\n" + "20000\n" + "30000\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class PartitionITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = PartitionProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object PartitionITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to PartitionProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
new file mode 100644
index 0000000..867eb81
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ReduceITCase.scala
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichReduceFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object ReduceProgs {
+ var NUM_PROGRAMS: Int = 10
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Reduce on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1)
+ .reduce { (in1, in2) => (in1._1 + in2._1, in1._2, "B-)") }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,B-)\n" + "15,3,B-)\n" + "34,4,B-)\n" + "65,5,B-)\n" + "111,6,B-)\n"
+
+ case 2 =>
+ /*
+ * Reduce on tuples with multiple key field selectors
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val reduceDs = ds.groupBy(4, 0)
+ .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
+ "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
+ "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
+
+ case 3 =>
+ /*
+ * Reduce on tuples with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(_._2)
+ .reduce { (in1, in2) => (in1._1 + in2._1, in1._2, "B-)") }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,B-)\n" + "15,3,B-)\n" + "34,4,B-)\n" + "65,5,B-)\n" + "111,6,B-)\n"
+
+ case 4 =>
+ /*
+ * Reduce on custom type with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds.groupBy(_.myInt)
+ .reduce { (in1, in2) =>
+ in1.myLong += in2.myLong
+ in1.myString = "Hello!"
+ in1
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,Hi\n" + "2,3,Hello!\n" + "3,12,Hello!\n" + "4,30,Hello!\n" + "5,60," +
+ "Hello!\n" + "6,105,Hello!\n"
+
+ case 5 =>
+ /*
+ * All-reduce for tuple
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs =
+ ds.reduce { (in1, in2) => (in1._1 + in2._1, in1._2 + in2._2, "Hello World") }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "231,91,Hello World\n"
+
+ case 6 =>
+ /*
+ * All-reduce for custom types
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds
+ .reduce { (in1, in2) =>
+ in1.myInt += in2.myInt
+ in1.myLong += in2.myLong
+ in1.myString = "Hello!"
+ in1
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "91,210,Hello!"
+
+ case 7 =>
+ /*
+ * Reduce with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).reduce(
+ new RichReduceFunction[(Int, Long, String)] {
+ private var f2Replace = ""
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ f2Replace = ints.sum + ""
+ }
+
+ override def reduce(
+ in1: (Int, Long, String),
+ in2: (Int, Long, String)): (Int, Long, String) = {
+ (in1._1 + in2._1, in1._2, f2Replace)
+ }
+ }).withBroadcastSet(intDs, "ints")
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,55\n" + "15,3,55\n" + "34,4,55\n" + "65,5,55\n" + "111,6,55\n"
+
+ case 8 =>
+ /*
+ * Reduce with UDF that returns the second input object (check mutable object handling)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env).map (t => MutableTuple3(t._1, t._2, t._3))
+ val reduceDs = ds.groupBy(1).reduce(
+ new RichReduceFunction[MutableTuple3[Int, Long, String]] {
+ override def reduce(
+ in1: MutableTuple3[Int, Long, String],
+ in2: MutableTuple3[Int, Long, String]): MutableTuple3[Int, Long, String] = {
+ in2._1 = in1._1 + in2._1
+ in2._3 = "Hi again!"
+ in2
+ }
+ })
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,Hi again!\n" + "15,3,Hi again!\n" + "34,4,Hi again!\n" + "65,5," +
+ "Hi again!\n" + "111,6,Hi again!\n"
+
+ case 9 =>
+ /*
+ * Reduce with a Tuple-returning KeySelector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val reduceDs = ds.groupBy(t => (t._1, t._5))
+ .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
+ "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
+ "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
+
+ case 10 =>
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+ val reduceDs = ds.groupBy("_5", "_1")
+ .reduce { (in1, in2) => (in1._1, in1._2 + in2._2, 0, "P-)", in1._5) }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,Hallo,1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,2,1,Hallo Welt,2\n" + "3,9,0," +
+ "P-),2\n" + "3,6,5,BCD,3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,10,GHI," +
+ "1\n" + "5,29,0,P-),2\n" + "5,25,0,P-),3\n"
+
+ case id =>
+ throw new IllegalArgumentException(s"Invalid program id $id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class ReduceITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = ReduceProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object ReduceITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to ReduceProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
new file mode 100644
index 0000000..488520e
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/SumMinMaxITCase.scala
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.java.aggregation.Aggregations
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+
+/**
+ * These tests are copied from [[AggregateITCase]] replacing calls to aggregate with calls to sum,
+ * min, and max
+ */
+object SumMinMaxProgs {
+ var NUM_PROGRAMS: Int = 3
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ // Full aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(10)
+ // val ds = CollectionDataSets.get3TupleDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .sum(0)
+ .andMax(1)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map{ t => (t._1, t._2) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "231,6\n"
+
+ case 2 =>
+ // Grouped aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .groupBy(1)
+ .sum(0)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map { t => (t._2, t._1) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n"
+
+ case 3 =>
+ // Nested aggregate
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+
+ val aggregateDs = ds
+ .groupBy(1)
+ .min(0)
+ .min(0)
+ // Ensure aggregate operator correctly copies other fields
+ .filter(_._3 != null)
+ .map { t => new Tuple1(t._1) }
+
+ aggregateDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1\n"
+
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class SumMinMaxITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = SumMinMaxProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object SumMinMaxITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to SumMinMaxProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
new file mode 100644
index 0000000..bf29b58
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/UnionITCase.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.Assert
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object UnionProgs {
+ var NUM_PROGRAMS: Int = 3
+
+ private final val FULL_TUPLE_3_STRING: String = "1,1,Hi\n" + "2,2,Hello\n" + "3,2," +
+ "Hello world\n" + "4,3,Hello world, how are you?\n" + "5,3,I am fine.\n" + "6,3," +
+ "Luke Skywalker\n" + "7,4,Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4," +
+ "Comment#4\n" + "11,5,Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5," +
+ "Comment#8\n" + "15,5,Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6," +
+ "Comment#12\n" + "19,6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Union of 2 Same Data Sets
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env))
+ unionDs.writeAsCsv(resultPath)
+ env.execute()
+ FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING
+
+ case 2 =>
+ /*
+ * Union of 5 same Data Sets, with multiple unions
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val unionDs = ds
+ .union(CollectionDataSets.get3TupleDataSet(env))
+ .union(CollectionDataSets.get3TupleDataSet(env))
+ .union(CollectionDataSets.get3TupleDataSet(env))
+ .union(CollectionDataSets.get3TupleDataSet(env))
+ unionDs.writeAsCsv(resultPath)
+ env.execute()
+ FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING +
+ FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING
+
+ case 3 =>
+ /*
+ * Test on union with empty dataset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ // Don't know how to make an empty result in an other way than filtering it
+ val empty = CollectionDataSets.get3TupleDataSet(env).filter( t => false )
+ val unionDs = CollectionDataSets.get3TupleDataSet(env).union(empty)
+ unionDs.writeAsCsv(resultPath)
+ env.execute()
+ FULL_TUPLE_3_STRING
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class UnionITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = UnionProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object UnionITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to UnionProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
new file mode 100644
index 0000000..c4d7dc8
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators.translation
+
+import org.apache.flink.api.common.Plan
+import org.apache.flink.api.common.operators.GenericDataSourceBase
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase
+
+import org.apache.flink.api.java.aggregation.Aggregations
+import org.apache.flink.api.scala._
+import org.junit.Assert.{assertEquals, assertTrue, fail}
+import org.junit.Test
+
+class AggregateTranslationTest {
+ @Test
+ def translateAggregate(): Unit = {
+ try {
+ val DOP = 8
+
+ val env = ExecutionEnvironment.createLocalEnvironment(DOP)
+
+ val initialData = env.fromElements((3.141592, "foobar", 77L))
+
+ initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).print()
+
+ val p: Plan = env.createProgramPlan()
+ val sink = p.getDataSinks.iterator.next
+
+ val reducer= sink.getInput.asInstanceOf[GroupReduceOperatorBase[_, _, _]]
+
+ assertEquals(1, reducer.getKeyColumns(0).length)
+ assertEquals(0, reducer.getKeyColumns(0)(0))
+ assertEquals(-1, reducer.getDegreeOfParallelism)
+ assertTrue(reducer.isCombinable)
+ assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Test caused an error: " + e.getMessage)
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
new file mode 100644
index 0000000..4a60154
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DeltaIterationTranslationTest.scala
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators.translation
+
+import org.apache.flink.api.common.functions.{RichCoGroupFunction, RichMapFunction,
+RichJoinFunction}
+import org.apache.flink.api.common.operators.GenericDataSinkBase
+import org.apache.flink.api.java.operators.translation.WrappingFunction
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.fail
+import org.apache.flink.api.common.{InvalidProgramException, Plan}
+import org.apache.flink.api.common.aggregators.LongSumAggregator
+import org.apache.flink.api.common.operators.base.DeltaIterationBase
+import org.apache.flink.api.common.operators.base.JoinOperatorBase
+import org.apache.flink.api.common.operators.base.MapOperatorBase
+import org.junit.Test
+
+import org.apache.flink.util.Collector
+
+import org.apache.flink.api.scala._
+
+class DeltaIterationTranslationTest {
+
+ @Test
+ def testCorrectTranslation(): Unit = {
+ try {
+ val JOB_NAME = "Test JobName"
+ val ITERATION_NAME = "Test Name"
+ val BEFORE_NEXT_WORKSET_MAP = "Some Mapper"
+ val AGGREGATOR_NAME = "AggregatorName"
+ val ITERATION_KEYS = Array(2)
+ val NUM_ITERATIONS = 13
+ val DEFAULT_DOP = 133
+ val ITERATION_DOP = 77
+
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(DEFAULT_DOP)
+
+ val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
+ val initialWorkSet = env.fromElements((1.23, "abc"))
+
+ val result = initialSolutionSet.iterateDelta(initialWorkSet, NUM_ITERATIONS, ITERATION_KEYS) {
+ (s, ws) =>
+ val wsSelfJoin = ws.map(new IdentityMapper[(Double, String)]())
+ .join(ws).where(1).equalTo(1) { (l, r) => l }
+
+ val joined = wsSelfJoin.join(s).where(1).equalTo(2).apply(new SolutionWorksetJoin)
+ (joined, joined.map(new NextWorksetMapper).name(BEFORE_NEXT_WORKSET_MAP))
+ }
+ result.name(ITERATION_NAME)
+ .setParallelism(ITERATION_DOP)
+ .registerAggregator(AGGREGATOR_NAME, new LongSumAggregator)
+
+ result.print()
+ result.writeAsText("/dev/null")
+
+ val p: Plan = env.createProgramPlan(JOB_NAME)
+ assertEquals(JOB_NAME, p.getJobName)
+ assertEquals(DEFAULT_DOP, p.getDefaultParallelism)
+ var sink1: GenericDataSinkBase[_] = null
+ var sink2: GenericDataSinkBase[_] = null
+ val sinks = p.getDataSinks.iterator
+ sink1 = sinks.next
+ sink2 = sinks.next
+
+ val iteration: DeltaIterationBase[_, _] =
+ sink1.getInput.asInstanceOf[DeltaIterationBase[_,_]]
+
+ assertEquals(iteration, sink2.getInput)
+ assertEquals(NUM_ITERATIONS, iteration.getMaximumNumberOfIterations)
+ assertArrayEquals(ITERATION_KEYS, iteration.getSolutionSetKeyFields)
+ assertEquals(ITERATION_DOP, iteration.getDegreeOfParallelism)
+ assertEquals(ITERATION_NAME, iteration.getName)
+
+ val nextWorksetMapper: MapOperatorBase[_, _, _] =
+ iteration.getNextWorkset.asInstanceOf[MapOperatorBase[_, _, _]]
+ val solutionSetJoin: JoinOperatorBase[_, _, _, _] =
+ iteration.getSolutionSetDelta.asInstanceOf[JoinOperatorBase[_, _, _, _]]
+ val worksetSelfJoin: JoinOperatorBase[_, _, _, _] =
+ solutionSetJoin.getFirstInput.asInstanceOf[JoinOperatorBase[_, _, _, _]]
+ val worksetMapper: MapOperatorBase[_, _, _] =
+ worksetSelfJoin.getFirstInput.asInstanceOf[MapOperatorBase[_, _, _]]
+
+ assertEquals(classOf[IdentityMapper[_]], worksetMapper.getUserCodeWrapper.getUserCodeClass)
+
+
+ assertEquals(
+ classOf[NextWorksetMapper],
+ nextWorksetMapper.getUserCodeWrapper.getUserCodeClass)
+
+
+ if (solutionSetJoin.getUserCodeWrapper.getUserCodeObject.isInstanceOf[WrappingFunction[_]]) {
+ val wf: WrappingFunction[_] = solutionSetJoin.getUserCodeWrapper.getUserCodeObject
+ .asInstanceOf[WrappingFunction[_]]
+ assertEquals(classOf[SolutionWorksetJoin],
+ wf.getWrappedFunction.getClass)
+ }
+ else {
+ assertEquals(classOf[SolutionWorksetJoin],
+ solutionSetJoin.getUserCodeWrapper.getUserCodeClass)
+ }
+
+ assertEquals(BEFORE_NEXT_WORKSET_MAP, nextWorksetMapper.getName)
+ assertEquals(AGGREGATOR_NAME, iteration.getAggregators.getAllRegisteredAggregators.iterator
+ .next.getName)
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail(e.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def testRejectWhenSolutionSetKeysDontMatchJoin(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
+ val initialWorkSet = env.fromElements((1.23, "abc"))
+
+ val iteration = initialSolutionSet.iterateDelta(initialWorkSet, 10, Array(0)) {
+ (s, ws) =>
+ try {
+ ws.join(s).where(1).equalTo(2)
+ fail("Accepted invalid program.")
+ } catch {
+ case e: InvalidProgramException => // all good
+ }
+ try {
+ s.join(ws).where(2).equalTo(1)
+ fail("Accepted invalid program.")
+ } catch {
+ case e: InvalidProgramException => // all good
+ }
+ (s, ws)
+ }
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail(e.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def testRejectWhenSolutionSetKeysDontMatchCoGroup(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val initialSolutionSet = env.fromElements((3.44, 5L, "abc"))
+ val initialWorkSet = env.fromElements((1.23, "abc"))
+
+ val iteration = initialSolutionSet.iterateDelta(initialWorkSet, 10, Array(0)) {
+ (s, ws) =>
+ try {
+ ws.coGroup(s).where(1).equalTo(2)
+ fail("Accepted invalid program.")
+ } catch {
+ case e: InvalidProgramException => // all good
+ }
+ try {
+ s.coGroup(ws).where(2).equalTo(1)
+ fail("Accepted invalid program.")
+ } catch {
+ case e: InvalidProgramException => // all good
+ }
+ (s, ws)
+ }
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail(e.getMessage)
+ }
+ }
+ }
+//
+// @Test def testRejectWhenSolutionSetKeysDontMatchCoGroup {
+// try {
+// val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+// @SuppressWarnings(Array("unchecked")) val initialSolutionSet: DataSet[Tuple3[Double, Long,
+// String]] = env.fromElements(new Tuple3[Double, Long, String](3.44, 5L, "abc"))
+// @SuppressWarnings(Array("unchecked")) val initialWorkSet: DataSet[Tuple2[Double,
+// String]] = env.fromElements(new Tuple2[Double, String](1.23, "abc"))
+// val iteration: DeltaIteration[Tuple3[Double, Long, String], Tuple2[Double,
+// String]] = initialSolutionSet.iterateDelta(initialWorkSet, 10, 1)
+// try {
+// iteration.getWorkset.coGroup(iteration.getSolutionSet).where(1).equalTo(2).`with`(
+// new DeltaIterationTranslationTest.SolutionWorksetCoGroup1)
+// fail("Accepted invalid program.")
+// }
+// catch {
+// case e: InvalidProgramException => {
+// }
+// }
+// try {
+// iteration.getSolutionSet.coGroup(iteration.getWorkset).where(2).equalTo(1).`with`(
+// new DeltaIterationTranslationTest.SolutionWorksetCoGroup2)
+// fail("Accepted invalid program.")
+// }
+// catch {
+// case e: InvalidProgramException => {
+// }
+// }
+// }
+// catch {
+// case e: Exception => {
+// System.err.println(e.getMessage)
+// e.printStackTrace
+// fail(e.getMessage)
+// }
+// }
+// }
+}
+
+class SolutionWorksetJoin
+ extends RichJoinFunction[(Double, String), (Double, Long, String), (Double, Long, String)] {
+ def join(first: (Double, String), second: (Double, Long, String)): (Double, Long, String) = {
+ null
+ }
+}
+
+class NextWorksetMapper extends RichMapFunction[(Double, Long, String), (Double, String)] {
+ def map(value: (Double, Long, String)): (Double, String) = {
+ null
+ }
+}
+
+class IdentityMapper[T] extends RichMapFunction[T, T] {
+ def map(value: T): T = {
+ value
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
new file mode 100644
index 0000000..931d34d
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/DistinctTranslationTest.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators.translation
+
+import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase
+import org.junit.Assert
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class DistinctTranslationTest {
+ @Test
+ def testCombinable(): Unit = {
+ try {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val input = env.fromElements("1", "2", "1", "3")
+
+ val op = input.distinct { x => x}
+ op.print()
+
+ val p = env.createProgramPlan()
+
+ val reduceOp =
+ p.getDataSinks.iterator.next.getInput.asInstanceOf[GroupReduceOperatorBase[_, _, _]]
+
+ Assert.assertTrue(reduceOp.isCombinable)
+ }
+ catch {
+ case e: Exception => {
+ e.printStackTrace()
+ Assert.fail(e.getMessage)
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
new file mode 100644
index 0000000..22e4914
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/translation/ReduceTranslationTest.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators.translation
+
+import org.apache.flink.api.common.operators.{GenericDataSourceBase, GenericDataSinkBase}
+import org.apache.flink.api.java.operators.translation.{KeyExtractingMapper,
+PlanUnwrappingReduceOperator}
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.typeutils.TupleTypeInfo
+import org.junit.Assert._
+import org.apache.flink.api.common.operators.base.MapOperatorBase
+import org.apache.flink.api.common.operators.base.ReduceOperatorBase
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class ReduceTranslationTest {
+ @Test
+ def translateNonGroupedReduce(): Unit = {
+ try {
+ val DOP = 8
+ val env = ExecutionEnvironment.createLocalEnvironment(DOP)
+
+ val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
+
+
+ initialData reduce { (v1, v2) => v1 } print()
+
+ val p = env.createProgramPlan(
+
+)
+ val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
+ val reducer: ReduceOperatorBase[_, _] = sink.getInput.asInstanceOf[ReduceOperatorBase[_, _]]
+
+ assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getInputType)
+ assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getOutputType)
+ assertTrue(reducer.getKeyColumns(0) == null || reducer.getKeyColumns(0).length == 0)
+ assertTrue(reducer.getDegreeOfParallelism == 1 || reducer.getDegreeOfParallelism == -1)
+ assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Test caused an error: " + e.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def translateGroupedReduceNoMapper(): Unit = {
+ try {
+ val DOP: Int = 8
+ val env = ExecutionEnvironment.createLocalEnvironment(DOP)
+
+ val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
+
+ initialData.groupBy(2) reduce { (v1, v2) => v1 } print()
+
+ val p = env.createProgramPlan()
+
+ val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
+ val reducer: ReduceOperatorBase[_, _] = sink.getInput.asInstanceOf[ReduceOperatorBase[_, _]]
+ assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getInputType)
+ assertEquals(initialData.javaSet.getType, reducer.getOperatorInfo.getOutputType)
+ assertTrue(reducer.getDegreeOfParallelism == DOP || reducer.getDegreeOfParallelism == -1)
+ assertArrayEquals(Array[Int](2), reducer.getKeyColumns(0))
+ assertTrue(reducer.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Test caused an error: " + e.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def translateGroupedReduceWithKeyExtractor(): Unit = {
+ try {
+ val DOP: Int = 8
+ val env = ExecutionEnvironment.createLocalEnvironment(DOP)
+
+ val initialData = env.fromElements((3.141592, "foobar", 77L)).setParallelism(1)
+
+ initialData.groupBy { _._2 }. reduce { (v1, v2) => v1 } setParallelism(4) print()
+
+ val p = env.createProgramPlan()
+ val sink: GenericDataSinkBase[_] = p.getDataSinks.iterator.next
+ val keyProjector: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_,
+ _, _]]
+ val reducer: PlanUnwrappingReduceOperator[_, _] = keyProjector.getInput
+ .asInstanceOf[PlanUnwrappingReduceOperator[_, _]]
+ val keyExtractor: MapOperatorBase[_, _, _] = reducer.getInput
+ .asInstanceOf[MapOperatorBase[_, _, _]]
+ assertEquals(1, keyExtractor.getDegreeOfParallelism)
+ assertEquals(4, reducer.getDegreeOfParallelism)
+ assertEquals(4, keyProjector.getDegreeOfParallelism)
+ val keyValueInfo = new TupleTypeInfo(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ createTypeInformation[(Double, String, Long)])
+ assertEquals(initialData.javaSet.getType, keyExtractor.getOperatorInfo.getInputType)
+ assertEquals(keyValueInfo, keyExtractor.getOperatorInfo.getOutputType)
+ assertEquals(keyValueInfo, reducer.getOperatorInfo.getInputType)
+ assertEquals(keyValueInfo, reducer.getOperatorInfo.getOutputType)
+ assertEquals(keyValueInfo, keyProjector.getOperatorInfo.getInputType)
+ assertEquals(initialData.javaSet.getType, keyProjector.getOperatorInfo.getOutputType)
+ assertEquals(
+ classOf[KeyExtractingMapper[_, _]],
+ keyExtractor.getUserCodeWrapper.getUserCodeClass)
+ assertTrue(keyExtractor.getInput.isInstanceOf[GenericDataSourceBase[_, _]])
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Test caused an error: " + e.getMessage)
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
new file mode 100644
index 0000000..9fb0980
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/GenericPairComparatorTest.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{GenericPairComparator, TypeComparator, TypeSerializer}
+import org.apache.flink.api.common.typeutils.base.{DoubleComparator, DoubleSerializer, IntComparator, IntSerializer}
+
+import org.apache.flink.api.java.typeutils.runtime.TupleComparator
+import org.apache.flink.api.scala.runtime.tuple.base.PairComparatorTestBase
+import org.apache.flink.api.scala.typeutils.CaseClassComparator
+
+class GenericPairComparatorTest
+ extends PairComparatorTestBase[(Int, String, Double), (Int, Float, Long, Double)] {
+
+ override protected def createComparator(ascending: Boolean):
+ GenericPairComparator[(Int, String, Double), (Int, Float, Long, Double)] = {
+ val fields1 = Array[Int](0, 2)
+ val fields2 = Array[Int](0, 3)
+
+ val comps1 =
+ Array[TypeComparator[_]](new IntComparator(ascending), new DoubleComparator(ascending))
+ val comps2 =
+ Array[TypeComparator[_]](new IntComparator(ascending), new DoubleComparator(ascending))
+
+ val sers1 =
+ Array[TypeSerializer[_]](IntSerializer.INSTANCE, DoubleSerializer.INSTANCE)
+ val sers2 =
+ Array[TypeSerializer[_]](IntSerializer.INSTANCE, DoubleSerializer.INSTANCE)
+
+ val comp1 = new CaseClassComparator[(Int, String, Double)](fields1, comps1, sers1)
+ val comp2 = new CaseClassComparator[(Int, Float, Long, Double)](fields2, comps2, sers2)
+
+ new GenericPairComparator[(Int, String, Double), (Int, Float, Long, Double)](comp1, comp2)
+ }
+
+ protected def getSortedTestData:
+ (Array[(Int, String, Double)], Array[(Int, Float, Long, Double)]) = {
+ (dataISD, dataIDL)
+ }
+
+ private val dataISD = Array(
+ (4, "hello", 20.0),
+ (4, "world", 23.2),
+ (5, "hello", 18.0),
+ (5, "world", 19.2),
+ (6, "hello", 16.0),
+ (6, "world", 17.2),
+ (7, "hello", 14.0),
+ (7,"world", 15.2))
+
+ private val dataIDL = Array(
+ (4, 0.11f, 14L, 20.0),
+ (4, 0.221f, 15L, 23.2),
+ (5, 0.33f, 15L, 18.0),
+ (5, 0.44f, 20L, 19.2),
+ (6, 0.55f, 20L, 16.0),
+ (6, 0.66f, 29L, 17.2),
+ (7, 0.77f, 29L, 14.0),
+ (7, 0.88f, 34L, 15.2))
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
new file mode 100644
index 0000000..adc46ac
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.TypeComparator
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+import org.apache.flink.api.scala._
+
+
+class TupleComparatorILD2Test extends TupleComparatorTestBase[(Int, Long, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
+ .createComparator(Array(0, 1), Array(ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, Long, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, 14L, 20.0),
+ (4, 15L, 23.2),
+ (5, 15L, 20.0),
+ (5, 20L, 20.0),
+ (6, 20L, 23.2),
+ (6, 29L, 20.0),
+ (7, 29L, 20.0),
+ (7, 34L, 23.2)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
new file mode 100644
index 0000000..377f985
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+
+class TupleComparatorILD3Test extends TupleComparatorTestBase[(Int, Long, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
+ .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, Long, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, 4L, 20.0),
+ (4, 4L, 23.2),
+ (4, 9L, 20.0),
+ (5, 4L, 20.0),
+ (5, 4L, 23.2),
+ (5, 9L, 20.0),
+ (6, 4L, 20.0),
+ (6, 4L, 23.2)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
new file mode 100644
index 0000000..1578951
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+
+class TupleComparatorILDC3Test extends TupleComparatorTestBase[(Int, Long, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
+ .createComparator(Array(2, 0, 1), Array(ascending, ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, Long, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, 4L, 20.0),
+ (5, 1L, 20.0),
+ (5, 2L, 20.0),
+ (5, 10L, 23.0),
+ (5, 19L, 24.0),
+ (5, 20L, 24.0),
+ (5, 24L, 25.0),
+ (5, 25L, 25.0)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
new file mode 100644
index 0000000..51e08cc
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+import org.apache.flink.api.scala._
+
+class TupleComparatorILDX1Test extends TupleComparatorTestBase[(Int, Long, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
+ .createComparator(Array(1), Array(ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, Long, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, 4L, 20.0),
+ (4, 5L, 23.2),
+ (4, 9L, 20.0),
+ (4, 10L, 24.0),
+ (4, 19L, 23.2),
+ (4, 20L, 24.0),
+ (4, 24L, 20.0),
+ (4, 25L, 23.2)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
new file mode 100644
index 0000000..3dbaabf
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+import org.apache.flink.api.scala._
+
+class TupleComparatorILDXC2Test extends TupleComparatorTestBase[(Int, Long, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
+ .createComparator(Array(2, 1), Array(ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
+ val ti = createTypeInformation[(Int, Long, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, Long, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, 4L, 20.0),
+ (4, 5L, 20.0),
+ (4, 3L, 23.0),
+ (4, 19L, 23.0),
+ (4, 17L, 24.0),
+ (4, 18L, 24.0),
+ (4, 24L, 25.0),
+ (4, 25L, 25.0)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
new file mode 100644
index 0000000..252ae79
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+class TupleComparatorISD1Test extends TupleComparatorTestBase[(Int, String, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
+ .createComparator(Array(0), Array(ascending),0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, String, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, "hello", 20.0),
+ (5, "hello", 23.2),
+ (6, "world", 20.0),
+ (7, "hello", 20.0),
+ (8, "hello", 23.2),
+ (9, "world", 20.0),
+ (10, "hello", 20.0),
+ (11, "hello", 23.2)
+ )
+}
+
[2/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
new file mode 100644
index 0000000..37e775e
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+class TupleComparatorISD2Test extends TupleComparatorTestBase[(Int, String, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
+ .createComparator(Array(0, 1), Array(ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, String, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, "hello", 20.0),
+ (4, "world", 23.2),
+ (5, "hello", 20.0),
+ (5, "world", 20.0),
+ (6, "hello", 23.2),
+ (6, "world", 20.0),
+ (7, "hello", 20.0),
+ (7, "world", 23.2)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
new file mode 100644
index 0000000..227b041
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.common.typeutils.{TypeSerializer, TypeComparator}
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.runtime.tuple.base.TupleComparatorTestBase
+
+class TupleComparatorISD3Test extends TupleComparatorTestBase[(Int, String, Double)] {
+
+ protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
+ .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
+ }
+
+ protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
+ val ti = createTypeInformation[(Int, String, Double)]
+ ti.createSerializer()
+ }
+
+ protected def getSortedTestData: Array[(Int, String, Double)] = {
+ dataISD
+ }
+
+ private val dataISD = Array(
+ (4, "hello", 20.0),
+ (4, "hello", 23.2),
+ (4, "world", 20.0),
+ (5, "hello", 20.0),
+ (5, "hello", 23.2),
+ (5, "world", 20.0),
+ (6, "hello", 20.0),
+ (6, "hello", 23.2)
+ )
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
new file mode 100644
index 0000000..a4149dd
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTest.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.java.typeutils.runtime.AbstractGenericTypeSerializerTest._
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.util.StringUtils
+import org.junit.Assert
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+import scala.collection.JavaConverters._
+
+import java.util.Random
+
+class TupleSerializerTest {
+
+ @Test
+ def testTuple1Int(): Unit = {
+ val testTuples = Array(Tuple1(42), Tuple1(1), Tuple1(0), Tuple1(-1), Tuple1(Int.MaxValue),
+ Tuple1(Int.MinValue))
+ runTests(testTuples)
+ }
+
+ @Test
+ def testTuple1String(): Unit = {
+ val rnd: Random = new Random(68761564135413L)
+ val testTuples = Array(
+ Tuple1(StringUtils.getRandomString(rnd, 10, 100)),
+ Tuple1("abc"),
+ Tuple1(""),
+ Tuple1(StringUtils.getRandomString(rnd, 30, 170)),
+ Tuple1(StringUtils.getRandomString(rnd, 15, 50)),
+ Tuple1(""))
+ runTests(testTuples)
+ }
+
+ @Test
+ def testTuple1StringArray(): Unit = {
+ val rnd: Random = new Random(289347567856686223L)
+
+ val arr1 = Array(
+ "abc",
+ "",
+ StringUtils.getRandomString(rnd, 10, 100),
+ StringUtils.getRandomString(rnd, 15, 50),
+ StringUtils.getRandomString(rnd, 30, 170),
+ StringUtils.getRandomString(rnd, 14, 15),
+ "")
+ val arr2 = Array(
+ "foo",
+ "",
+ StringUtils.getRandomString(rnd, 10, 100),
+ StringUtils.getRandomString(rnd, 1000, 5000),
+ StringUtils.getRandomString(rnd, 30000, 35000),
+ StringUtils.getRandomString(rnd, 100 * 1024, 105 * 1024),
+ "bar")
+ val testTuples = Array(Tuple1(arr1), Tuple1(arr2))
+ runTests(testTuples)
+ }
+
+ @Test
+ def testTuple2StringDouble(): Unit = {
+ val rnd: Random = new Random(807346528946L)
+
+ val testTuples = Array(
+ (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
+ (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
+ (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
+ ("", rnd.nextDouble),
+ (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble),
+ (StringUtils.getRandomString(rnd, 10, 100), rnd.nextDouble))
+ runTests(testTuples)
+ }
+
+ @Test
+ def testTuple2StringStringArray(): Unit = {
+ val rnd: Random = new Random(289347567856686223L)
+
+ val arr1 = Array(
+ "abc",
+ "",
+ StringUtils.getRandomString(rnd, 10, 100),
+ StringUtils.getRandomString(rnd, 15, 50),
+ StringUtils.getRandomString(rnd, 30, 170),
+ StringUtils.getRandomString(rnd, 14, 15), "")
+ val arr2 = Array(
+ "foo",
+ "",
+ StringUtils.getRandomString(rnd, 10, 100),
+ StringUtils.getRandomString(rnd, 1000, 5000),
+ StringUtils.getRandomString(rnd, 30000, 35000),
+ StringUtils.getRandomString(rnd, 100 * 1024, 105 * 1024),
+ "bar")
+ val testTuples = Array(
+ (StringUtils.getRandomString(rnd, 30, 170), arr1),
+ (StringUtils.getRandomString(rnd, 30, 170), arr2),
+ (StringUtils.getRandomString(rnd, 30, 170), arr1),
+ (StringUtils.getRandomString(rnd, 30, 170), arr2),
+ (StringUtils.getRandomString(rnd, 30, 170), arr2))
+ runTests(testTuples)
+ }
+
+ @Test
+ def testTuple5CustomObjects(): Unit = {
+ val rnd: Random = new Random(807346528946L)
+
+ val a = new SimpleTypes
+ val b = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+ val c = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+ val d = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+ val e = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+ val f = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+ val g = new SimpleTypes(rnd.nextInt, rnd.nextLong, rnd.nextInt.asInstanceOf[Byte],
+ StringUtils.getRandomString(rnd, 10, 100), rnd.nextInt.asInstanceOf[Short], rnd.nextDouble)
+
+ val o1 = new ComplexNestedObject1(5626435)
+ val o2 = new ComplexNestedObject1(76923)
+ val o3 = new ComplexNestedObject1(-1100)
+ val o4 = new ComplexNestedObject1(0)
+ val o5 = new ComplexNestedObject1(44)
+
+ val co1 = new ComplexNestedObject2(rnd)
+ val co2 = new ComplexNestedObject2
+ val co3 = new ComplexNestedObject2(rnd)
+ val co4 = new ComplexNestedObject2(rnd)
+
+ val b1 = new Book(976243875L, "The Serialization Odysse", 42)
+ val b2 = new Book(0L, "Debugging byte streams", 1337)
+ val b3 = new Book(-1L, "Low level interfaces", 0xC0FFEE)
+ val b4 = new Book(Long.MaxValue, "The joy of bits and bytes", 0xDEADBEEF)
+ val b5 = new Book(Long.MaxValue, "Winnign a prize for creative test strings", 0xBADF00)
+ val b6 = new Book(-2L, "Distributed Systems", 0xABCDEF0123456789L)
+
+ val list = List("A", "B", "C", "D", "E")
+ val ba1 = new BookAuthor(976243875L, list.asJava, "Arno Nym")
+
+ val list2 = List[String]()
+ val ba2 = new BookAuthor(987654321L, list2.asJava, "The Saurus")
+
+ val testTuples = Array(
+ (a, b1, o1, ba1, co1),
+ (b, b2, o2, ba2, co2),
+ (c, b3, o3, ba1, co3),
+ (d, b2, o4, ba1, co4),
+ (e, b4, o5, ba2, co4),
+ (f, b5, o1, ba2, co4),
+ (g, b6, o4, ba1, co2))
+ runTests(testTuples)
+ }
+
+ private final def runTests[T <: Product : TypeInformation](instances: Array[T]) {
+ try {
+ val tupleTypeInfo = implicitly[TypeInformation[T]].asInstanceOf[TupleTypeInfoBase[T]]
+ val serializer = tupleTypeInfo.createSerializer
+ val tupleClass = tupleTypeInfo.getTypeClass
+ val test = new TupleSerializerTestInstance[T](serializer, tupleClass, -1, instances)
+ test.testAll()
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ Assert.fail(e.getMessage)
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
new file mode 100644
index 0000000..3b923bd
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/TupleSerializerTestInstance.scala
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime
+
+import org.junit.Assert._
+import org.apache.flink.api.common.typeutils.SerializerTestInstance
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.junit.Assert
+import org.junit.Test
+
+
+class TupleSerializerTestInstance[T <: Product] (
+ serializer: TypeSerializer[T],
+ typeClass: Class[T],
+ length: Int,
+ testData: Array[T])
+ extends SerializerTestInstance[T](serializer, typeClass, length, testData: _*) {
+
+ @Test
+ override def testInstantiate(): Unit = {
+ try {
+ val serializer: TypeSerializer[T] = getSerializer
+ val instance: T = serializer.createInstance
+ assertNotNull("The created instance must not be null.", instance)
+ val tpe: Class[T] = getTypeClass
+ assertNotNull("The test is corrupt: type class is null.", tpe)
+ // We cannot check this because Tuple1 instances are not actually of type Tuple1
+ // but something like Tuple1$mcI$sp
+// assertEquals("Type of the instantiated object is wrong.", tpe, instance.getClass)
+ }
+ catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+
+ protected override def deepEquals(message: String, shouldTuple: T, isTuple: T) {
+ Assert.assertEquals(shouldTuple.productArity, isTuple.productArity)
+ for (i <- 0 until shouldTuple.productArity) {
+ val should = shouldTuple.productElement(i)
+ val is = isTuple.productElement(i)
+ if (should.getClass.isArray) {
+ should match {
+ case booleans: Array[Boolean] =>
+ Assert.assertTrue(message, booleans.sameElements(is.asInstanceOf[Array[Boolean]]))
+ case bytes: Array[Byte] =>
+ assertArrayEquals(message, bytes, is.asInstanceOf[Array[Byte]])
+ case shorts: Array[Short] =>
+ assertArrayEquals(message, shorts, is.asInstanceOf[Array[Short]])
+ case ints: Array[Int] =>
+ assertArrayEquals(message, ints, is.asInstanceOf[Array[Int]])
+ case longs: Array[Long] =>
+ assertArrayEquals(message, longs, is.asInstanceOf[Array[Long]])
+ case floats: Array[Float] =>
+ assertArrayEquals(message, floats, is.asInstanceOf[Array[Float]], 0.0f)
+ case doubles: Array[Double] =>
+ assertArrayEquals(message, doubles, is.asInstanceOf[Array[Double]], 0.0)
+ case chars: Array[Char] =>
+ assertArrayEquals(message, chars, is.asInstanceOf[Array[Char]])
+ case _ =>
+ assertArrayEquals(
+ message,
+ should.asInstanceOf[Array[AnyRef]],
+ is.asInstanceOf[Array[AnyRef]])
+ }
+ } else {
+ assertEquals(message, should, is)
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
new file mode 100644
index 0000000..9809dd2
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/PairComparatorTestBase.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime.tuple.base
+
+import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
+import org.apache.flink.api.common.typeutils.TypePairComparator
+import org.junit.Test
+
+/**
+ * Abstract test base for PairComparators.
+ */
+abstract class PairComparatorTestBase[T, R] {
+ protected def createComparator(ascending: Boolean): TypePairComparator[T, R]
+
+ protected def getSortedTestData: (Array[T], Array[R])
+
+ @Test
+ def testEqualityWithReference(): Unit = {
+ try {
+ val comparator = getComparator(true)
+
+ val (dataT, dataR) = getSortedData
+ for (i <- 0 until dataT.length) {
+ comparator.setReference(dataT(i))
+ assertTrue(comparator.equalToReference(dataR(i)))
+ }
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+
+ @Test
+ def testInequalityWithReference(): Unit = {
+ testGreatSmallAscDescWithReference(true)
+ testGreatSmallAscDescWithReference(false)
+ }
+
+ protected def testGreatSmallAscDescWithReference(ascending: Boolean) {
+ try {
+ val (dataT, dataR) = getSortedData
+ val comparator = getComparator(ascending)
+ for (x <- 0 until (dataT.length - 1)) {
+ for (y <- (x + 1) until dataR.length) {
+ comparator.setReference(dataT(x))
+ if (ascending) {
+ assertTrue(comparator.compareToReference(dataR(y)) > 0)
+ }
+ else {
+ assertTrue(comparator.compareToReference(dataR(y)) < 0)
+ }
+ }
+ }
+ } catch {
+ case e: Exception => {
+ System.err.println(e.getMessage)
+ e.printStackTrace()
+ fail("Exception in test: " + e.getMessage)
+ }
+ }
+ }
+
+ protected def getComparator(ascending: Boolean): TypePairComparator[T, R] = {
+ val comparator: TypePairComparator[T, R] = createComparator(ascending)
+ if (comparator == null) {
+ throw new RuntimeException("Test case corrupt. Returns null as comparator.")
+ }
+ comparator
+ }
+
+ protected def getSortedData: (Array[T], Array[R]) = {
+ val (dataT, dataR) = getSortedTestData
+
+ if (dataT == null || dataR == null) {
+ throw new RuntimeException("Test case corrupt. Returns null as test data.")
+ }
+ if (dataT.length < 2 || dataR.length < 2) {
+ throw new RuntimeException("Test case does not provide enough sorted test data.")
+ }
+ (dataT, dataR)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
new file mode 100644
index 0000000..506b264
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/runtime/tuple/base/TupleComparatorTestBase.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.runtime.tuple.base
+
+import org.apache.flink.api.common.typeutils.ComparatorTestBase
+import org.apache.flink.api.scala.typeutils.{CaseClassSerializer, CaseClassComparator}
+import org.junit.Assert._
+
+abstract class TupleComparatorTestBase[T <: Product] extends ComparatorTestBase[T] {
+ protected override def deepEquals(message: String, should: T, is: T) {
+ for (i <- 0 until should.productArity) {
+ assertEquals(should.productElement(i), is.productElement(i))
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
new file mode 100644
index 0000000..2b2d3a9
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.types
+
+import java.io.DataInput
+import java.io.DataOutput
+import org.apache.flink.api.common.typeinfo._
+import org.apache.flink.api.java.typeutils._
+import org.apache.flink.types.{IntValue, StringValue}
+import org.apache.hadoop.io.Writable
+import org.junit.Assert
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+class MyWritable extends Writable {
+ def write(out: DataOutput) {
+ }
+
+ def readFields(in: DataInput) {
+ }
+}
+
+case class CustomCaseClass(a: String, b: Int)
+
+class CustomType(var myField1: String, var myField2: Int) {
+ def this() {
+ this(null, 0)
+ }
+}
+
+class MyObject[A](var a: A) {
+ def this() { this(null.asInstanceOf[A]) }
+}
+
+class TypeInformationGenTest {
+
+ @Test
+ def testBasicType(): Unit = {
+ val ti = createTypeInformation[Boolean]
+
+ Assert.assertTrue(ti.isBasicType)
+ Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, ti)
+ Assert.assertEquals(classOf[java.lang.Boolean], ti.getTypeClass)
+ }
+
+ @Test
+ def testWritableType(): Unit = {
+ val ti = createTypeInformation[MyWritable]
+
+ Assert.assertTrue(ti.isInstanceOf[WritableTypeInfo[_]])
+ Assert.assertEquals(classOf[MyWritable], ti.asInstanceOf[WritableTypeInfo[_]].getTypeClass)
+ }
+
+ @Test
+ def testTupleWithBasicTypes(): Unit = {
+ val ti = createTypeInformation[(Int, Long, Double, Float, Boolean, String, Char, Short, Byte)]
+
+ Assert.assertTrue(ti.isTupleType)
+ Assert.assertEquals(9, ti.getArity)
+ Assert.assertTrue(ti.isInstanceOf[TupleTypeInfoBase[_]])
+ val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
+ Assert.assertEquals(classOf[Tuple9[_,_,_,_,_,_,_,_,_]], tti.getTypeClass)
+ for (i <- 0 until 0) {
+ Assert.assertTrue(tti.getTypeAt(i).isInstanceOf[BasicTypeInfo[_]])
+ }
+
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, tti.getTypeAt(0))
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tti.getTypeAt(1))
+ Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, tti.getTypeAt(2))
+ Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, tti.getTypeAt(3))
+ Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, tti.getTypeAt(4))
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(5))
+ Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, tti.getTypeAt(6))
+ Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, tti.getTypeAt(7))
+ Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, tti.getTypeAt(8))
+ }
+
+ @Test
+ def testTupleWithTuples(): Unit = {
+ val ti = createTypeInformation[(Tuple1[String], Tuple1[Int], Tuple2[Long, Long])]
+
+ Assert.assertTrue(ti.isTupleType())
+ Assert.assertEquals(3, ti.getArity)
+ Assert.assertTrue(ti.isInstanceOf[TupleTypeInfoBase[_]])
+ val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
+ Assert.assertEquals(classOf[Tuple3[_, _, _]], tti.getTypeClass)
+ Assert.assertTrue(tti.getTypeAt(0).isTupleType())
+ Assert.assertTrue(tti.getTypeAt(1).isTupleType())
+ Assert.assertTrue(tti.getTypeAt(2).isTupleType())
+ Assert.assertEquals(classOf[Tuple1[_]], tti.getTypeAt(0).getTypeClass)
+ Assert.assertEquals(classOf[Tuple1[_]], tti.getTypeAt(1).getTypeClass)
+ Assert.assertEquals(classOf[Tuple2[_, _]], tti.getTypeAt(2).getTypeClass)
+ Assert.assertEquals(1, tti.getTypeAt(0).getArity)
+ Assert.assertEquals(1, tti.getTypeAt(1).getArity)
+ Assert.assertEquals(2, tti.getTypeAt(2).getArity)
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO,
+ tti.getTypeAt(0).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO,
+ tti.getTypeAt(1).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO,
+ tti.getTypeAt(2).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO,
+ tti.getTypeAt(2).asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1))
+ }
+
+ @Test
+ def testCaseClass(): Unit = {
+ val ti = createTypeInformation[CustomCaseClass]
+
+ Assert.assertTrue(ti.isTupleType)
+ Assert.assertEquals(2, ti.getArity)
+ Assert.assertEquals(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0))
+ Assert.assertEquals(
+ BasicTypeInfo.INT_TYPE_INFO,
+ ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1))
+ Assert.assertEquals(
+ classOf[CustomCaseClass],ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeClass())
+ }
+
+ @Test
+ def testCustomType(): Unit = {
+ val ti = createTypeInformation[CustomType]
+
+ Assert.assertFalse(ti.isBasicType)
+ Assert.assertFalse(ti.isTupleType)
+ Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
+ Assert.assertEquals(ti.getTypeClass, classOf[CustomType])
+ }
+
+ @Test
+ def testTupleWithCustomType(): Unit = {
+ val ti = createTypeInformation[(Long, CustomType)]
+
+ Assert.assertTrue(ti.isTupleType)
+ Assert.assertEquals(2, ti.getArity)
+ val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
+ Assert.assertEquals(classOf[Tuple2[_, _]], tti.getTypeClass)
+ Assert.assertEquals(classOf[java.lang.Long], tti.getTypeAt(0).getTypeClass)
+ Assert.assertTrue(tti.getTypeAt(1).isInstanceOf[PojoTypeInfo[_]])
+ Assert.assertEquals(classOf[CustomType], tti.getTypeAt(1).getTypeClass)
+ }
+
+ @Test
+ def testValue(): Unit = {
+ val ti = createTypeInformation[StringValue]
+
+ Assert.assertFalse(ti.isBasicType)
+ Assert.assertFalse(ti.isTupleType)
+ Assert.assertTrue(ti.isInstanceOf[ValueTypeInfo[_]])
+ Assert.assertEquals(ti.getTypeClass, classOf[StringValue])
+ Assert.assertTrue(TypeExtractor.getForClass(classOf[StringValue])
+ .isInstanceOf[ValueTypeInfo[_]])
+ Assert.assertEquals(TypeExtractor.getForClass(classOf[StringValue]).getTypeClass,
+ ti.getTypeClass)
+ }
+
+ @Test
+ def testTupleOfValues(): Unit = {
+ val ti = createTypeInformation[(StringValue, IntValue)]
+ Assert.assertFalse(ti.isBasicType)
+ Assert.assertTrue(ti.isTupleType)
+ Assert.assertEquals(
+ classOf[StringValue],
+ ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(0).getTypeClass)
+ Assert.assertEquals(
+ classOf[IntValue],
+ ti.asInstanceOf[TupleTypeInfoBase[_]].getTypeAt(1).getTypeClass)
+ }
+
+
+ @Test
+ def testBasicArray(): Unit = {
+ val ti = createTypeInformation[Array[String]]
+
+ Assert.assertFalse(ti.isBasicType)
+ Assert.assertFalse(ti.isTupleType)
+ Assert.assertTrue(ti.isInstanceOf[BasicArrayTypeInfo[_, _]] ||
+ ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
+ if (ti.isInstanceOf[BasicArrayTypeInfo[_, _]]) {
+ Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, ti)
+ }
+ else {
+ Assert.assertEquals(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ ti.asInstanceOf[ObjectArrayTypeInfo[_, _]].getComponentInfo)
+ }
+ }
+
+ @Test
+ def testPrimitiveArray(): Unit = {
+ val ti = createTypeInformation[Array[Boolean]]
+
+ Assert.assertTrue(ti.isInstanceOf[PrimitiveArrayTypeInfo[_]])
+ Assert.assertEquals(ti, PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO)
+ }
+
+ @Test
+ def testCustomArray(): Unit = {
+ val ti = createTypeInformation[Array[CustomType]]
+ Assert.assertTrue(ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
+ Assert.assertEquals(
+ classOf[CustomType],
+ ti.asInstanceOf[ObjectArrayTypeInfo[_, _]].getComponentType)
+ }
+
+ @Test
+ def testTupleArray(): Unit = {
+ val ti = createTypeInformation[Array[(String, String)]]
+
+ Assert.assertTrue(ti.isInstanceOf[ObjectArrayTypeInfo[_, _]])
+ val oati = ti.asInstanceOf[ObjectArrayTypeInfo[_, _]]
+ Assert.assertTrue(oati.getComponentInfo.isTupleType)
+ val tti = oati.getComponentInfo.asInstanceOf[TupleTypeInfoBase[_]]
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(0))
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tti.getTypeAt(1))
+ }
+
+ @Test
+ def testParamertizedCustomObject(): Unit = {
+ val ti = createTypeInformation[MyObject[String]]
+
+ Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
+ }
+
+ @Test
+ def testTupleWithPrimitiveArray(): Unit = {
+ val ti = createTypeInformation[(Array[Int], Array[Double], Array[Long],
+ Array[Byte], Array[Char], Array[Float], Array[Short], Array[Boolean],
+ Array[String])]
+
+ val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
+ Assert.assertEquals(PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(0))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(1))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(2))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(3))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(4))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(5))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(6))
+ Assert.assertEquals(PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO, tti.getTypeAt(7))
+ Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, tti.getTypeAt(8))
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
new file mode 100644
index 0000000..09e049b
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.util
+
+import org.apache.hadoop.io.IntWritable
+
+import org.apache.flink.api.scala._
+
+import scala.collection.mutable
+import scala.reflect.classTag
+import scala.util.Random
+
+/**
+ * #################################################################################################
+ *
+ * BE AWARE THAT OTHER TESTS DEPEND ON THIS TEST DATA.
+ * IF YOU MODIFY THE DATA MAKE SURE YOU CHECK THAT ALL TESTS ARE STILL WORKING!
+ *
+ * #################################################################################################
+ */
+object CollectionDataSets {
+ def get3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ data.+=((4, 3L, "Hello world, how are you?"))
+ data.+=((5, 3L, "I am fine."))
+ data.+=((6, 3L, "Luke Skywalker"))
+ data.+=((7, 4L, "Comment#1"))
+ data.+=((8, 4L, "Comment#2"))
+ data.+=((9, 4L, "Comment#3"))
+ data.+=((10, 4L, "Comment#4"))
+ data.+=((11, 5L, "Comment#5"))
+ data.+=((12, 5L, "Comment#6"))
+ data.+=((13, 5L, "Comment#7"))
+ data.+=((14, 5L, "Comment#8"))
+ data.+=((15, 5L, "Comment#9"))
+ data.+=((16, 6L, "Comment#10"))
+ data.+=((17, 6L, "Comment#11"))
+ data.+=((18, 6L, "Comment#12"))
+ data.+=((19, 6L, "Comment#13"))
+ data.+=((20, 6L, "Comment#14"))
+ data.+=((21, 6L, "Comment#15"))
+ Random.shuffle(data)
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmall3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def get5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
+ val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
+ data.+=((1, 1L, 0, "Hallo", 1L))
+ data.+=((2, 2L, 1, "Hallo Welt", 2L))
+ data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
+ data.+=((3, 4L, 3, "Hallo Welt wie gehts?", 2L))
+ data.+=((3, 5L, 4, "ABC", 2L))
+ data.+=((3, 6L, 5, "BCD", 3L))
+ data.+=((4, 7L, 6, "CDE", 2L))
+ data.+=((4, 8L, 7, "DEF", 1L))
+ data.+=((4, 9L, 8, "EFG", 1L))
+ data.+=((4, 10L, 9, "FGH", 2L))
+ data.+=((5, 11L, 10, "GHI", 1L))
+ data.+=((5, 12L, 11, "HIJ", 3L))
+ data.+=((5, 13L, 12, "IJK", 3L))
+ data.+=((5, 14L, 13, "JKL", 2L))
+ data.+=((5, 15L, 14, "KLM", 2L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmall5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
+ val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
+ data.+=((1, 1L, 0, "Hallo", 1L))
+ data.+=((2, 2L, 1, "Hallo Welt", 2L))
+ data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
+ val data = new mutable.MutableList[((Int, Int), String)]
+ data.+=(((1, 1), "one"))
+ data.+=(((2, 2), "two"))
+ data.+=(((3, 3), "three"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getGroupSortedNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
+ val data = new mutable.MutableList[((Int, Int), String)]
+ data.+=(((1, 3), "a"))
+ data.+=(((1, 2), "a"))
+ data.+=(((2, 1), "a"))
+ data.+=(((2, 2), "b"))
+ data.+=(((3, 3), "c"))
+ data.+=(((3, 6), "c"))
+ data.+=(((4, 9), "c"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getStringDataSet(env: ExecutionEnvironment): DataSet[String] = {
+ val data = new mutable.MutableList[String]
+ data.+=("Hi")
+ data.+=("Hello")
+ data.+=("Hello world")
+ data.+=("Hello world, how are you?")
+ data.+=("I am fine.")
+ data.+=("Luke Skywalker")
+ data.+=("Random comment")
+ data.+=("LOL")
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getIntDataSet(env: ExecutionEnvironment): DataSet[Int] = {
+ val data = new mutable.MutableList[Int]
+ data.+=(1)
+ data.+=(2)
+ data.+=(2)
+ data.+=(3)
+ data.+=(3)
+ data.+=(3)
+ data.+=(4)
+ data.+=(4)
+ data.+=(4)
+ data.+=(4)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
+ val data = new mutable.MutableList[CustomType]
+ data.+=(new CustomType(1, 0L, "Hi"))
+ data.+=(new CustomType(2, 1L, "Hello"))
+ data.+=(new CustomType(2, 2L, "Hello world"))
+ data.+=(new CustomType(3, 3L, "Hello world, how are you?"))
+ data.+=(new CustomType(3, 4L, "I am fine."))
+ data.+=(new CustomType(3, 5L, "Luke Skywalker"))
+ data.+=(new CustomType(4, 6L, "Comment#1"))
+ data.+=(new CustomType(4, 7L, "Comment#2"))
+ data.+=(new CustomType(4, 8L, "Comment#3"))
+ data.+=(new CustomType(4, 9L, "Comment#4"))
+ data.+=(new CustomType(5, 10L, "Comment#5"))
+ data.+=(new CustomType(5, 11L, "Comment#6"))
+ data.+=(new CustomType(5, 12L, "Comment#7"))
+ data.+=(new CustomType(5, 13L, "Comment#8"))
+ data.+=(new CustomType(5, 14L, "Comment#9"))
+ data.+=(new CustomType(6, 15L, "Comment#10"))
+ data.+=(new CustomType(6, 16L, "Comment#11"))
+ data.+=(new CustomType(6, 17L, "Comment#12"))
+ data.+=(new CustomType(6, 18L, "Comment#13"))
+ data.+=(new CustomType(6, 19L, "Comment#14"))
+ data.+=(new CustomType(6, 20L, "Comment#15"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
+ val data = new mutable.MutableList[CustomType]
+ data.+=(new CustomType(1, 0L, "Hi"))
+ data.+=(new CustomType(2, 1L, "Hello"))
+ data.+=(new CustomType(2, 2L, "Hello world"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallTuplebasedPojoMatchingDataSet(env: ExecutionEnvironment):
+ DataSet[(Int, String, Int, Int, Long, String, Long)] = {
+ val data = new mutable.MutableList[(Int, String, Int, Int, Long, String, Long)]
+ data.+=((1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=((2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=((3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallPojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
+ val data = new mutable.MutableList[POJO]
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getDuplicatePojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
+ val data = new mutable.MutableList[POJO]
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(data)
+ }
+
+ def getCrazyNestedDataSet(env: ExecutionEnvironment): DataSet[CrazyNested] = {
+ val data = new mutable.MutableList[CrazyNested]
+ data.+=(new CrazyNested("aa"))
+ data.+=(new CrazyNested("bb"))
+ data.+=(new CrazyNested("bb"))
+ data.+=(new CrazyNested("cc"))
+ data.+=(new CrazyNested("cc"))
+ data.+=(new CrazyNested("cc"))
+ env.fromCollection(data)
+ }
+
+ def getPojoContainingTupleAndWritable(env: ExecutionEnvironment): DataSet[CollectionDataSets
+ .PojoContainingTupleAndWritable] = {
+ val data = new
+ mutable.MutableList[PojoContainingTupleAndWritable]
+ data.+=(new PojoContainingTupleAndWritable(1, 10L, 100L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ env.fromCollection(data)
+ }
+
+ def getGroupSortedPojoContainingTupleAndWritable(env: ExecutionEnvironment):
+ DataSet[PojoContainingTupleAndWritable] = {
+ val data = new mutable.MutableList[PojoContainingTupleAndWritable]
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(1, 10L, 100L))
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 20L, 201L))
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 200L))
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 600L))
+ data.+=(new CollectionDataSets.PojoContainingTupleAndWritable(2, 30L, 400L))
+ env.fromCollection(data)
+ }
+
+ def getTupleContainingPojos(env: ExecutionEnvironment): DataSet[(Int, CrazyNested, POJO)] = {
+ val data = new mutable.MutableList[(Int, CrazyNested, POJO)]
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 2,
+ new CrazyNested("two", "duo", 2L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ env.fromCollection(data)
+ }
+
+ def getSmallTuplebasedDataSetMatchingPojo(env: ExecutionEnvironment):
+ DataSet[(Long, Integer, Integer, Long, String, Integer, String)] = {
+ val data = new mutable.MutableList[(Long, Integer, Integer, Long, String, Integer, String)]
+ data.+=((10000L, 10, 100, 1000L, "One", 1, "First"))
+ data.+=((20000L, 20, 200, 2000L, "Two", 2, "Second"))
+ data.+=((30000L, 30, 300, 3000L, "Three", 3, "Third"))
+ env.fromCollection(data)
+ }
+
+ def getPojoWithMultiplePojos(env: ExecutionEnvironment): DataSet[CollectionDataSets
+ .PojoWithMultiplePojos] = {
+ val data = new mutable.MutableList[CollectionDataSets
+ .PojoWithMultiplePojos]
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("a", "aa", "b", "bb", 1))
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("d", "dd", "e", "ee", 3))
+ data.+=(new CollectionDataSets.PojoWithMultiplePojos("d", "dd", "e", "ee", 3))
+ env.fromCollection(data)
+ }
+
+ case class MutableTuple3[T1, T2, T3](var _1: T1, var _2: T2, var _3: T3)
+
+ class CustomType(var myInt: Int, var myLong: Long, var myString: String) {
+ def this() {
+ this(0, 0, "")
+ }
+
+ override def toString: String = {
+ myInt + "," + myLong + "," + myString
+ }
+ }
+
+ class POJO(
+ var number: Int,
+ var str: String,
+ var nestedTupleWithCustom: (Int, CustomType),
+ var nestedPojo: NestedPojo) {
+ def this() {
+ this(0, "", null, null)
+ }
+
+ def this(i0: Int, s0: String, i1: Int, i2: Int, l0: Long, s1: String, l1: Long) {
+ this(i0, s0, (i1, new CustomType(i2, l0, s1)), new NestedPojo(l1))
+ }
+
+ override def toString: String = {
+ number + " " + str + " " + nestedTupleWithCustom + " " + nestedPojo.longNumber
+ }
+
+ @transient var ignoreMe: Long = 1L
+ }
+
+ class NestedPojo(var longNumber: Long) {
+ def this() {
+ this(0)
+ }
+ }
+
+ class CrazyNested(var nest_Lvl1: CrazyNestedL1, var something: Long) {
+ def this() {
+ this(new CrazyNestedL1, 0)
+ }
+
+ def this(set: String) {
+ this()
+ nest_Lvl1 = new CrazyNestedL1
+ nest_Lvl1.nest_Lvl2 = new CrazyNestedL2
+ nest_Lvl1.nest_Lvl2.nest_Lvl3 = new CrazyNestedL3
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4 = new CrazyNestedL4
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal = set
+ }
+
+ def this(set: String, second: String, s: Long) {
+ this(set)
+ something = s
+ nest_Lvl1.a = second
+ }
+ }
+
+ class CrazyNestedL1 {
+ var a: String = null
+ var b: Int = 0
+ var nest_Lvl2: CrazyNestedL2 = null
+ }
+
+ class CrazyNestedL2 {
+ var nest_Lvl3: CrazyNestedL3 = null
+ }
+
+ class CrazyNestedL3 {
+ var nest_Lvl4: CrazyNestedL4 = null
+ }
+
+ class CrazyNestedL4 {
+ var f1nal: String = null
+ }
+
+ class PojoContainingTupleAndWritable(
+ var someInt: Int,
+ var someString: String,
+ var hadoopFan: IntWritable,
+ var theTuple: (Long, Long)) {
+ def this() {
+ this(0, "", new IntWritable(0), (0, 0))
+ }
+
+ def this(i: Int, l1: Long, l2: Long) {
+ this()
+ hadoopFan = new IntWritable(i)
+ someInt = i
+ theTuple = (l1, l2)
+ }
+
+ }
+
+ class Pojo1 {
+ var a: String = null
+ var b: String = null
+
+ override def toString = s"Pojo1 a=$a b=$b"
+ }
+
+ class Pojo2 {
+ var a2: String = null
+ var b2: String = null
+
+ override def toString = s"Pojo2 a2=$a2 b2=$b2"
+ }
+
+ class PojoWithMultiplePojos {
+
+ def this(a: String, b: String, a1: String, b1: String, i0: Int) {
+ this()
+ p1 = new Pojo1
+ p1.a = a
+ p1.b = b
+ p2 = new Pojo2
+ p2.a2 = a1
+ p2.b2 = b1
+ this.i0 = i0
+ }
+
+ var p1: Pojo1 = null
+ var p2: Pojo2 = null
+ var i0: Int = 0
+
+ override def toString = s"PojoWithMultiplePojos p1=$p1 p2=$p2 i0=$i0"
+ }
+
+}
+
[4/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
new file mode 100644
index 0000000..f43052a
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.apache.flink.api.scala._
+import org.junit.runners.Parameterized.Parameters
+import scala.collection.JavaConverters._
+
+import scala.collection.mutable
+
+// TODO case class Tuple2[T1, T2](_1: T1, _2: T2)
+// TODO case class Foo(a: Int, b: String
+
+case class Nested(myLong: Long)
+
+class Pojo(var myString: String, var myInt: Int, var nested: Nested) {
+ def this() = {
+ this("", 0, new Nested(1))
+ }
+
+ def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
+
+ override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
+}
+
+class NestedPojo(var myLong: Long) {
+ def this() { this(0) }
+}
+
+class PojoWithPojo(var myString: String, var myInt: Int, var nested: Nested) {
+ def this() = {
+ this("", 0, new Nested(1))
+ }
+
+ def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
+
+ override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
+}
+
+object ExampleProgs {
+ var NUM_PROGRAMS: Int = 4
+
+ def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
+ progId match {
+ case 1 =>
+ /*
+ Test nested tuples with int offset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
+
+ val grouped = ds.groupBy(0).reduce( { (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)})
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "((this,hello),3)\n((this,is),3)\n"
+
+ case 2 =>
+ /*
+ Test nested tuples with int offset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
+
+ val grouped = ds.groupBy("_1._1").reduce{
+ (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "((this,is),6)\n"
+
+ case 3 =>
+ /*
+ Test nested pojos
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements(
+ new PojoWithPojo("one", 1, 1L),
+ new PojoWithPojo("one", 1, 1L),
+ new PojoWithPojo("two", 666, 2L) )
+
+ val grouped = ds.groupBy("nested.myLong").reduce {
+ (p1, p2) =>
+ p1.myInt += p2.myInt
+ p1
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
+
+ case 4 =>
+ /*
+ Test pojo with nested case class
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements(
+ new Pojo("one", 1, 1L),
+ new Pojo("one", 1, 1L),
+ new Pojo("two", 666, 2L) )
+
+ val grouped = ds.groupBy("nested.myLong").reduce {
+ (p1, p2) =>
+ p1.myInt += p2.myInt
+ p1
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
+ }
+ }
+}
+
+@RunWith(classOf[Parameterized])
+class ExamplesITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = ExampleProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object ExamplesITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to ExampleProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
new file mode 100644
index 0000000..973028b
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichFilterFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object FilterProgs {
+ var NUM_PROGRAMS: Int = 7
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Test all-rejecting filter.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val filterDs = ds.filter( t => false )
+ filterDs.writeAsCsv(resultPath)
+ env.execute()
+ "\n"
+
+ case 2 =>
+ /*
+ * Test all-passing filter.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val filterDs = ds.filter( t => true )
+ filterDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
+ "how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" + "7,4," +
+ "Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" + "11,5," +
+ "Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
+ "Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
+ "6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
+
+ case 3 =>
+ /*
+ * Test filter on String tuple field.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val filterDs = ds.filter( _._3.contains("world") )
+ filterDs.writeAsCsv(resultPath)
+ env.execute()
+ "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
+
+ case 4 =>
+ /*
+ * Test filter on Integer tuple field.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val filterDs = ds.filter( _._1 % 2 == 0 )
+ filterDs.writeAsCsv(resultPath)
+ env.execute()
+ "2,2,Hello\n" + "4,3,Hello world, how are you?\n" + "6,3,Luke Skywalker\n" + "8,4," +
+ "Comment#2\n" + "10,4,Comment#4\n" + "12,5,Comment#6\n" + "14,5,Comment#8\n" + "16,6," +
+ "Comment#10\n" + "18,6,Comment#12\n" + "20,6,Comment#14\n"
+
+ case 5 =>
+ /*
+ * Test filter on basic type
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getStringDataSet(env)
+ val filterDs = ds.filter( _.startsWith("H") )
+ filterDs.writeAsText(resultPath)
+ env.execute()
+ "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n"
+
+ case 6 =>
+ /*
+ * Test filter on custom type
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val filterDs = ds.filter( _.myString.contains("a") )
+ filterDs.writeAsText(resultPath)
+ env.execute()
+ "3,3,Hello world, how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n"
+
+ case 7 =>
+ /*
+ * Test filter on String tuple field.
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ints = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val filterDs = ds.filter( new RichFilterFunction[(Int, Long, String)] {
+ var literal = -1
+ override def open(config: Configuration): Unit = {
+ val ints = getRuntimeContext.getBroadcastVariable[Int]("ints")
+ for (i <- ints.asScala) {
+ literal = if (literal < i) i else literal
+ }
+ }
+ override def filter(value: (Int, Long, String)): Boolean = value._1 < literal
+ }).withBroadcastSet(ints, "ints")
+ filterDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class FilterITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = FilterProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object FilterITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to FilterProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
new file mode 100644
index 0000000..6882885
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.operators.Order
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object FirstNProgs {
+ var NUM_PROGRAMS: Int = 3
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * First-n on ungrouped data set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val seven = ds.first(7).map( t => new Tuple1(1) ).sum(0)
+ seven.writeAsText(resultPath)
+ env.execute()
+ "(7)\n"
+
+ case 2 =>
+ /*
+ * First-n on grouped data set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val first = ds.groupBy(1).first(4).map( t => (t._2, 1)).groupBy(0).sum(1)
+ first.writeAsText(resultPath)
+ env.execute()
+ "(1,1)\n(2,2)\n(3,3)\n(4,4)\n(5,4)\n(6,4)\n"
+
+ case 3 =>
+ /*
+ * First-n on grouped and sorted data set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val first = ds.groupBy(1)
+ .sortGroup(0, Order.DESCENDING)
+ .first(3)
+ .map ( t => (t._2, t._1))
+ first.writeAsText(resultPath)
+ env.execute()
+ "(1,1)\n" + "(2,3)\n(2,2)\n" + "(3,6)\n(3,5)\n(3,4)\n" + "(4,10)\n(4,9)\n(4," +
+ "8)\n" + "(5,15)\n(5,14)\n(5,13)\n" + "(6,21)\n(6,20)\n(6,19)\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class FirstNITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = FirstNProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object FirstNITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to FirstNProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
new file mode 100644
index 0000000..7c259b8
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.InvalidProgramException
+import org.apache.flink.api.common.operators.Order
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.junit.{Assert, Test}
+
+import org.apache.flink.api.scala._
+
+class FirstNOperatorTest {
+
+ private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+
+ @Test
+ def testUngroupedFirstN(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ try {
+ tupleDs.first(1)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.first(10)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.first(0)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.first(-1)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ }
+
+ @Test
+ def testGroupedFirstN(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ try {
+ tupleDs.groupBy(2).first(1)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(1, 3).first(10)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(0).first(0)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(2).first(-1)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ }
+
+ @Test
+ def testGroupedSortedFirstN(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ try {
+ tupleDs.groupBy(2).sortGroup(4, Order.ASCENDING).first(1)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(1, 3).sortGroup(4, Order.ASCENDING).first(10)
+ }
+ catch {
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(0).sortGroup(4, Order.ASCENDING).first(0)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ try {
+ tupleDs.groupBy(2).sortGroup(4, Order.ASCENDING).first(-1)
+ Assert.fail()
+ }
+ catch {
+ case ipe: InvalidProgramException => {
+ }
+ case e: Exception => {
+ Assert.fail()
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
new file mode 100644
index 0000000..0d80d22
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object FlatMapProgs {
+ var NUM_PROGRAMS: Int = 7
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Test non-passing flatmap
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getStringDataSet(env)
+ val nonPassingFlatMapDs = ds.flatMap( in => if (in.contains("banana")) Some(in) else None )
+ nonPassingFlatMapDs.writeAsText(resultPath)
+ env.execute()
+ "\n"
+
+ case 2 =>
+ /*
+ * Test data duplicating flatmap
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getStringDataSet(env)
+ val duplicatingFlatMapDs = ds.flatMap( in => Seq(in, in.toUpperCase) )
+ duplicatingFlatMapDs.writeAsText(resultPath)
+ env.execute()
+ "Hi\n" + "HI\n" + "Hello\n" + "HELLO\n" + "Hello world\n" + "HELLO WORLD\n" +
+ "Hello world, how are you?\n" + "HELLO WORLD, HOW ARE YOU?\n" + "I am fine.\n" + "I AM " +
+ "FINE.\n" + "Luke Skywalker\n" + "LUKE SKYWALKER\n" + "Random comment\n" + "RANDOM " +
+ "COMMENT\n" + "LOL\n" + "LOL\n"
+
+ case 3 =>
+ /*
+ * Test flatmap with varying number of emitted tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val varyingTuplesMapDs = ds.flatMap {
+ in =>
+ val numTuples = in._1 % 3
+ (0 until numTuples) map { i => in }
+ }
+ varyingTuplesMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "2,2,Hello\n" + "4,3,Hello world, " +
+ "how are you?\n" + "5,3,I am fine.\n" + "5,3,I am fine.\n" + "7,4,Comment#1\n" + "8,4," +
+ "Comment#2\n" + "8,4,Comment#2\n" + "10,4,Comment#4\n" + "11,5,Comment#5\n" + "11,5," +
+ "Comment#5\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "14,5,Comment#8\n" + "16,6," +
+ "Comment#10\n" + "17,6,Comment#11\n" + "17,6,Comment#11\n" + "19,6,Comment#13\n" + "20," +
+ "6,Comment#14\n" + "20,6,Comment#14\n"
+
+ case 4 =>
+ /*
+ * Test type conversion flatmapper (Custom -> Tuple)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val typeConversionFlatMapDs = ds.flatMap { in => Some((in.myInt, in.myLong, in.myString)) }
+ typeConversionFlatMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,0,Hi\n" + "2,1,Hello\n" + "2,2,Hello world\n" + "3,3,Hello world, " +
+ "how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n" + "4,6," +
+ "Comment#1\n" + "4,7,Comment#2\n" + "4,8,Comment#3\n" + "4,9,Comment#4\n" + "5,10," +
+ "Comment#5\n" + "5,11,Comment#6\n" + "5,12,Comment#7\n" + "5,13,Comment#8\n" + "5,14," +
+ "Comment#9\n" + "6,15,Comment#10\n" + "6,16,Comment#11\n" + "6,17,Comment#12\n" + "6," +
+ "18,Comment#13\n" + "6,19,Comment#14\n" + "6,20,Comment#15\n"
+
+ case 5 =>
+ /*
+ * Test type conversion flatmapper (Tuple -> Basic)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val typeConversionFlatMapDs = ds.flatMap ( in => Some(in._3) )
+ typeConversionFlatMapDs.writeAsText(resultPath)
+ env.execute()
+ "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
+ ".\n" + "Luke Skywalker\n" + "Comment#1\n" + "Comment#2\n" + "Comment#3\n" +
+ "Comment#4\n" + "Comment#5\n" + "Comment#6\n" + "Comment#7\n" + "Comment#8\n" +
+ "Comment#9\n" + "Comment#10\n" + "Comment#11\n" + "Comment#12\n" + "Comment#13\n" +
+ "Comment#14\n" + "Comment#15\n"
+
+ case 6 =>
+ /*
+ * Test flatmapper if UDF returns input object
+ * multiple times and changes it in between
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env).map {
+ t => MutableTuple3(t._1, t._2, t._3)
+ }
+ val inputObjFlatMapDs = ds.flatMap {
+ (in, out: Collector[MutableTuple3[Int, Long, String]]) =>
+ val numTuples = in._1 % 4
+ (0 until numTuples) foreach { i => in._1 = i; out.collect(in) }
+ }
+ inputObjFlatMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "0,1,Hi\n" + "0,2,Hello\n" + "1,2,Hello\n" + "0,2,Hello world\n" + "1,2," +
+ "Hello world\n" + "2,2,Hello world\n" + "0,3,I am fine.\n" + "0,3," +
+ "Luke Skywalker\n" + "1,3,Luke Skywalker\n" + "0,4,Comment#1\n" + "1,4," +
+ "Comment#1\n" + "2,4,Comment#1\n" + "0,4,Comment#3\n" + "0,4,Comment#4\n" + "1,4," +
+ "Comment#4\n" + "0,5,Comment#5\n" + "1,5,Comment#5\n" + "2,5,Comment#5\n" + "0,5," +
+ "Comment#7\n" + "0,5,Comment#8\n" + "1,5,Comment#8\n" + "0,5,Comment#9\n" + "1,5," +
+ "Comment#9\n" + "2,5,Comment#9\n" + "0,6,Comment#11\n" + "0,6,Comment#12\n" + "1,6," +
+ "Comment#12\n" + "0,6,Comment#13\n" + "1,6,Comment#13\n" + "2,6,Comment#13\n" + "0,6," +
+ "Comment#15\n"
+
+ case 7 =>
+ /*
+ * Test flatmap with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ints = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env).map {
+ t => MutableTuple3(t._1, t._2, t._3)
+ }
+ val bcFlatMapDs = ds.flatMap(
+ new RichFlatMapFunction[MutableTuple3[Int, Long, String],
+ MutableTuple3[Int, Long, String]] {
+ private var f2Replace = 0
+ private val outTuple = MutableTuple3(0, 0L, "")
+ override def open(config: Configuration): Unit = {
+ val ints = getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ f2Replace = ints.sum
+ }
+ override def flatMap(
+ value: MutableTuple3[Int, Long, String],
+ out: Collector[MutableTuple3[Int, Long, String]]): Unit = {
+ outTuple._1 = f2Replace
+ outTuple._2 = value._2
+ outTuple._3 = value._3
+ out.collect(outTuple)
+ }
+ }).withBroadcastSet(ints, "ints")
+ bcFlatMapDs.writeAsCsv(resultPath)
+ env.execute()
+ "55,1,Hi\n" + "55,2,Hello\n" + "55,2,Hello world\n" + "55,3,Hello world, " +
+ "how are you?\n" + "55,3,I am fine.\n" + "55,3,Luke Skywalker\n" + "55,4," +
+ "Comment#1\n" + "55,4,Comment#2\n" + "55,4,Comment#3\n" + "55,4,Comment#4\n" + "55,5," +
+ "Comment#5\n" + "55,5,Comment#6\n" + "55,5,Comment#7\n" + "55,5,Comment#8\n" + "55,5," +
+ "Comment#9\n" + "55,6,Comment#10\n" + "55,6,Comment#11\n" + "55,6,Comment#12\n" + "55," +
+ "6,Comment#13\n" + "55,6,Comment#14\n" + "55,6,Comment#15\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class FlatMapITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = FlatMapProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object FlatMapITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to FlatMapProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
new file mode 100644
index 0000000..7b81933
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
@@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions._
+import org.apache.flink.api.common.operators.Order
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.{CrazyNested, POJO, MutableTuple3,
+CustomType}
+import org.apache.flink.compiler.PactCompiler
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object GroupReduceProgs {
+ var NUM_PROGRAMS: Int = 26
+
+ def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
+ progId match {
+ case 1 =>
+ /*
+ * check correctness of groupReduce on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).reduceGroup {
+ in =>
+ in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
+
+ case 2 =>
+ /*
+ * check correctness of groupReduce on tuples with multiple key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .get5TupleDataSet(env)
+ val reduceDs = ds.groupBy(4, 0).reduceGroup {
+ in =>
+ val (i, l, l2) = in
+ .map( t => (t._1, t._2, t._5))
+ .reduce((l, r) => (l._1, l._2 + r._2, l._3))
+ (i, l, 0, "P-)", l2)
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
+ "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
+ "2\n" + "5,25,0,P-),3\n"
+
+ case 3 =>
+ /*
+ * check correctness of groupReduce on tuples with key field selector and group sorting
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).sortGroup(2, Order.ASCENDING).reduceGroup {
+ in =>
+ in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" +
+ "5,2,Hello-Hello world\n" +
+ "15,3,Hello world, how are you?-I am fine.-Luke Skywalker\n" +
+ "34,4,Comment#1-Comment#2-Comment#3-Comment#4\n" +
+ "65,5,Comment#5-Comment#6-Comment#7-Comment#8-Comment#9\n" +
+ "111,6,Comment#10-Comment#11-Comment#12-Comment#13-Comment#14-Comment#15\n"
+
+ case 4 =>
+ /*
+ * check correctness of groupReduce on tuples with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(_._2).reduceGroup {
+ in =>
+ in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
+
+ case 5 =>
+ /*
+ * check correctness of groupReduce on custom type with type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds.groupBy(_.myInt).reduceGroup {
+ in =>
+ val iter = in.toIterator
+ val o = new CustomType
+ val c = iter.next()
+
+ o.myString = "Hello!"
+ o.myInt = c.myInt
+ o.myLong = c.myLong
+
+ while (iter.hasNext) {
+ val next = iter.next()
+ o.myLong += next.myLong
+ }
+ o
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,Hello!\n" + "2,3,Hello!\n" + "3,12,Hello!\n" + "4,30,Hello!\n" + "5,60," +
+ "Hello!\n" + "6,105,Hello!\n"
+
+ case 6 =>
+ /*
+ * check correctness of all-groupreduce for tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.reduceGroup {
+ in =>
+ var i = 0
+ var l = 0L
+ for (t <- in) {
+ i += t._1
+ l += t._2
+ }
+ (i, l, "Hello World")
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "231,91,Hello World\n"
+
+ case 7 =>
+ /*
+ * check correctness of all-groupreduce for custom types
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds.reduceGroup {
+ in =>
+ val o = new CustomType(0, 0, "Hello!")
+ for (t <- in) {
+ o.myInt += t.myInt
+ o.myLong += t.myLong
+ }
+ o
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "91,210,Hello!"
+
+ case 8 =>
+ /*
+ * check correctness of groupReduce with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).reduceGroup(
+ new RichGroupReduceFunction[(Int, Long, String), (Int, Long, String)] {
+ private var f2Replace = ""
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ f2Replace = ints.sum + ""
+ }
+
+ override def reduce(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, Long, String)]): Unit = {
+ var i: Int = 0
+ var l: Long = 0L
+ for (t <- values.asScala) {
+ i += t._1
+ l = t._2
+ }
+ out.collect((i, l, f2Replace))
+ }
+ }).withBroadcastSet(intDs, "ints")
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,55\n" + "5,2,55\n" + "15,3,55\n" + "34,4,55\n" + "65,5,55\n" + "111,6,55\n"
+
+ case 9 =>
+ /*
+ * check correctness of groupReduce if UDF returns input objects multiple times and
+ * changes it in between
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ .map( t => MutableTuple3(t._1, t._2, t._3) )
+ val reduceDs = ds.groupBy(1).reduceGroup {
+ (in, out: Collector[MutableTuple3[Int, Long, String]]) =>
+ for (t <- in) {
+ if (t._1 < 4) {
+ t._3 = "Hi!"
+ t._1 += 10
+ out.collect(t)
+ t._1 += 10
+ t._3 = "Hi again!"
+ out.collect(t)
+ }
+ }
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "11,1,Hi!\n" + "21,1,Hi again!\n" + "12,2,Hi!\n" + "22,2,Hi again!\n" + "13,2," +
+ "Hi!\n" + "23,2,Hi again!\n"
+
+ case 10 =>
+ /*
+ * check correctness of groupReduce on custom type with key extractor and combine
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+
+ @RichGroupReduceFunction.Combinable
+ class CustomTypeGroupReduceWithCombine
+ extends RichGroupReduceFunction[CustomType, CustomType] {
+ override def combine(values: Iterable[CustomType], out: Collector[CustomType]): Unit = {
+ val o = new CustomType()
+ for (c <- values.asScala) {
+ o.myInt = c.myInt
+ o.myLong += c.myLong
+ o.myString = "test" + c.myInt
+ }
+ out.collect(o)
+ }
+
+ override def reduce(values: Iterable[CustomType], out: Collector[CustomType]): Unit = {
+ val o = new CustomType(0, 0, "")
+ for (c <- values.asScala) {
+ o.myInt = c.myInt
+ o.myLong += c.myLong
+ o.myString = c.myString
+ }
+ out.collect(o)
+ }
+ }
+ val reduceDs = ds.groupBy(_.myInt).reduceGroup(new CustomTypeGroupReduceWithCombine)
+
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ if (onCollection) {
+ null
+ }
+ else {
+ "1,0,test1\n" + "2,3,test2\n" + "3,12,test3\n" + "4,30,test4\n" + "5,60," +
+ "test5\n" + "6,105,test6\n"
+ }
+
+ case 11 =>
+ /*
+ * check correctness of groupReduce on tuples with combine
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ // important because it determines how often the combiner is called
+ env.setDegreeOfParallelism(2)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ @RichGroupReduceFunction.Combinable
+ class Tuple3GroupReduceWithCombine
+ extends RichGroupReduceFunction[(Int, Long, String), (Int, String)] {
+ override def combine(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, Long, String)]): Unit = {
+ var i = 0
+ var l = 0L
+ var s = ""
+ for (t <- values.asScala) {
+ i += t._1
+ l = t._2
+ s = "test" + t._2
+ }
+ out.collect((i, l, s))
+ }
+
+ override def reduce(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, String)]): Unit = {
+ var i = 0
+ var s = ""
+ for (t <- values.asScala) {
+ i += t._1
+ s = t._3
+ }
+ out.collect((i, s))
+ }
+ }
+ val reduceDs = ds.groupBy(1).reduceGroup(new Tuple3GroupReduceWithCombine)
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ if (onCollection) {
+ null
+ }
+ else {
+ "1,test1\n" + "5,test2\n" + "15,test3\n" + "34,test4\n" + "65,test5\n" + "111," +
+ "test6\n"
+ }
+
+
+ // all-groupreduce with combine
+
+
+ case 12 =>
+ /*
+ * check correctness of all-groupreduce for tuples with combine
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env).map(t => t).setParallelism(4)
+
+ val cfg: Configuration = new Configuration
+ cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)
+
+ @RichGroupReduceFunction.Combinable
+ class Tuple3AllGroupReduceWithCombine
+ extends RichGroupReduceFunction[(Int, Long, String), (Int, String)] {
+ override def combine(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, Long, String)]): Unit = {
+ var i = 0
+ var l = 0L
+ var s = ""
+ for (t <- values.asScala) {
+ i += t._1
+ l += t._2
+ s += "test"
+ }
+ out.collect((i, l, s))
+ }
+
+ override def reduce(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, String)]): Unit = {
+ var i = 0
+ var s = ""
+ for (t <- values.asScala) {
+ i += t._1 + t._2.toInt
+ s += t._3
+ }
+ out.collect((i, s))
+ }
+ }
+ val reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine).withParameters(cfg)
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ if (onCollection) {
+ null
+ }
+ else {
+ "322," +
+ "testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest\n"
+ }
+
+ case 13 =>
+ /*
+ * check correctness of groupReduce with descending group sort
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).sortGroup(2, Order.DESCENDING).reduceGroup {
+ in =>
+ in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
+ }
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
+ "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
+ "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
+ "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
+
+ case 14 =>
+ /*
+ * check correctness of groupReduce on tuples with tuple-returning key selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .get5TupleDataSet(env)
+ val reduceDs = ds.groupBy( t => (t._1, t._5)).reduceGroup {
+ in =>
+ val (i, l, l2) = in
+ .map( t => (t._1, t._2, t._5))
+ .reduce((l, r) => (l._1, l._2 + r._2, l._3))
+ (i, l, 0, "P-)", l2)
+ }
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
+ "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
+ "2\n" + "5,25,0,P-),3\n"
+
+ case 15 =>
+ /*
+ * check that input of combiner is also sorted for combinable groupReduce with group
+ * sorting
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.get3TupleDataSet(env).map { t =>
+ MutableTuple3(t._1, t._2, t._3)
+ }
+
+ @RichGroupReduceFunction.Combinable
+ class OrderCheckingCombinableReduce
+ extends RichGroupReduceFunction[MutableTuple3[Int, Long, String],
+ MutableTuple3[Int, Long, String]] {
+ def reduce(
+ values: Iterable[MutableTuple3[Int, Long, String]],
+ out: Collector[MutableTuple3[Int, Long, String]]) {
+ val it = values.iterator()
+ var t = it.next()
+ val i = t._1
+ out.collect(t)
+
+ while (it.hasNext) {
+ t = it.next()
+ if (i > t._1 || (t._3 == "INVALID-ORDER!")) {
+ t._3 = "INVALID-ORDER!"
+ out.collect(t)
+ }
+ }
+ }
+
+ override def combine(
+ values: Iterable[MutableTuple3[Int, Long, String]],
+ out: Collector[MutableTuple3[Int, Long, String]]) {
+ val it = values.iterator()
+ var t = it.next
+ val i: Int = t._1
+ out.collect(t)
+ while (it.hasNext) {
+ t = it.next
+ if (i > t._1) {
+ t._3 = "INVALID-ORDER!"
+ out.collect(t)
+ }
+ }
+ }
+ }
+
+ val reduceDs = ds.groupBy(1)
+ .sortGroup(0, Order.ASCENDING).reduceGroup(new OrderCheckingCombinableReduce)
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "4,3,Hello world, how are you?\n" + "7,4," +
+ "Comment#1\n" + "11,5,Comment#5\n" + "16,6,Comment#10\n"
+
+ case 16 =>
+ /*
+ * Deep nesting test
+ * + null value in pojo
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .getCrazyNestedDataSet(env)
+ val reduceDs = ds.groupBy("nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal")
+ .reduceGroup {
+ in =>
+ var c = 0
+ var n: String = null
+ for (v <- in) {
+ c += 1
+ n = v.nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal
+ }
+ (n, c)
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "aa,1\nbb,2\ncc,3\n"
+
+
+ case 17 =>
+ // We don't have that test but keep numbering compatible to Java GroupReduceITCase
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.fromElements("Hello world").writeAsText(resultPath)
+ env.execute()
+ "Hello world"
+
+ case 18 =>
+ /*
+ * Test Pojo containing a Writable and Tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .getPojoContainingTupleAndWritable(env)
+ val reduceDs = ds.groupBy("hadoopFan", "theTuple.*").reduceGroup(new
+ GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, Integer] {
+ def reduce(
+ values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
+ out: Collector[Integer]) {
+ var c: Int = 0
+ for (v <- values.asScala) {
+ c += 1
+ }
+ out.collect(c)
+ }
+ })
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1\n5\n"
+
+ case 19 =>
+ /*
+ * Test Tuple containing pojos and regular fields
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getTupleContainingPojos(env)
+ val reduceDs = ds.groupBy("_1", "_2.*").reduceGroup(
+ new GroupReduceFunction[(Int, CrazyNested, POJO), Int] {
+ def reduce(values: Iterable[(Int, CrazyNested, POJO)], out: Collector[Int]) {
+ var c: Int = 0
+ for (v <- values.asScala) {
+ c += 1
+ }
+ out.collect(c)
+ }
+ })
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "3\n1\n"
+
+ case 20 =>
+ /*
+ * Test string-based definition on group sort, based on test:
+ * check correctness of groupReduce with descending group sort
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1)
+ .sortGroup("_3", Order.DESCENDING)
+ .reduceGroup {
+ in =>
+ in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
+ "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
+ "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
+ "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
+
+ case 21 =>
+ /*
+ * Test int-based definition on group sort, for (full) nested Tuple
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
+ val reduceDs = ds.groupBy("_2").sortGroup(0, Order.DESCENDING)
+ .reduceGroup(new NestedTupleReducer)
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"
+
+
+ case 22 =>
+ /*
+ * Test int-based definition on group sort, for (partial) nested Tuple ASC
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
+ val reduceDs = ds.groupBy("_2")
+ .sortGroup("_1._1", Order.ASCENDING)
+ .sortGroup("_1._2", Order.ASCENDING)
+ .reduceGroup(new NestedTupleReducer)
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "a--(1,2)-(1,3)-(2,1)-\n" + "b--(2,2)-\n" + "c--(3,3)-(3,6)-(4,9)-\n"
+
+ case 23 =>
+ /*
+ * Test string-based definition on group sort, for (partial) nested Tuple DESC
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
+ val reduceDs = ds.groupBy("_2")
+ .sortGroup("_1._1", Order.DESCENDING)
+ .sortGroup("_1._2", Order.ASCENDING)
+ .reduceGroup(new NestedTupleReducer)
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "a--(2,1)-(1,2)-(1,3)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,3)-(3,6)-\n"
+
+ case 24 =>
+ /*
+ * Test string-based definition on group sort, for two grouping keys
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
+ val reduceDs = ds.groupBy("_2")
+ .sortGroup("_1._1", Order.DESCENDING)
+ .sortGroup("_1._2", Order.DESCENDING)
+ .reduceGroup(new NestedTupleReducer)
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"
+
+ case 25 =>
+ /*
+ * Test string-based definition on group sort, for two grouping keys with Pojos
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getGroupSortedPojoContainingTupleAndWritable(env)
+ val reduceDs = ds.groupBy("hadoopFan")
+ .sortGroup("theTuple._1", Order.DESCENDING)
+ .sortGroup("theTuple._2", Order.DESCENDING)
+ .reduceGroup(
+ new GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, String] {
+ def reduce(
+ values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
+ out: Collector[String]) {
+ var once: Boolean = false
+ val concat: StringBuilder = new StringBuilder
+ for (value <- values.asScala) {
+ if (!once) {
+ concat.append(value.hadoopFan.get)
+ concat.append("---")
+ once = true
+ }
+ concat.append(value.theTuple)
+ concat.append("-")
+ }
+ out.collect(concat.toString())
+ }
+ })
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1---(10,100)-\n" + "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n"
+
+ case 26 =>
+ /*
+ * Test grouping with pojo containing multiple pojos (was a bug)
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.getPojoWithMultiplePojos(env)
+ val reduceDs = ds.groupBy("p2.a2")
+ .reduceGroup(
+ new GroupReduceFunction[CollectionDataSets.PojoWithMultiplePojos, String] {
+ def reduce(
+ values: Iterable[CollectionDataSets.PojoWithMultiplePojos],
+ out: Collector[String]) {
+ val concat: StringBuilder = new StringBuilder
+ for (value <- values.asScala) {
+ concat.append(value.p2.a2)
+ }
+ out.collect(concat.toString())
+ }
+ })
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "b\nccc\nee\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class GroupReduceITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private val curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = GroupReduceProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ }
+
+ protected override def postSubmit(): Unit = {
+ if (expectedResult != null) compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object GroupReduceITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to GroupReduceProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
+class NestedTupleReducer extends GroupReduceFunction[((Int, Int), String), String] {
+ def reduce(values: Iterable[((Int, Int), String)], out: Collector[String]) {
+ var once: Boolean = false
+ val concat: StringBuilder = new StringBuilder
+ for (value <- values.asScala) {
+ if (!once) {
+ concat.append(value._2).append("--")
+ once = true
+ }
+ concat.append(value._1)
+ concat.append("-")
+ }
+ out.collect(concat.toString())
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
new file mode 100644
index 0000000..fe1dd43
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.junit.Assert
+import org.apache.flink.api.common.InvalidProgramException
+import org.apache.flink.api.common.operators.Order
+import org.junit.Test
+
+import org.apache.flink.api.scala._
+
+
+class GroupingTest {
+
+ private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
+ private val customTypeData = Array[CustomType](new CustomType())
+ private val emptyLongData = Array[Long]()
+
+ @Test
+ def testGroupByKeyIndices1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ tupleDs.groupBy(0)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testGroupByKeyIndices2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work, grouping on basic type
+ longDs.groupBy(0)
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testGroupByKeyIndices3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work, field position key on custom type
+ customDs.groupBy(0)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyIndices4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, fiels position out of range
+ tupleDs.groupBy(5)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyIndices5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, negative field position
+ tupleDs.groupBy(-1)
+ }
+
+ @Test
+ def testGroupByKeyFields1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should work
+ try {
+ tupleDs.groupBy("_1")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyFields2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val longDs = env.fromCollection(emptyLongData)
+
+ // should not work, grouping on basic type
+ longDs.groupBy("_1")
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyFields3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work, field key on custom type
+ customDs.groupBy("_1")
+ }
+
+ @Test(expected = classOf[RuntimeException])
+ def testGroupByKeyFields4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, invalid field
+ tupleDs.groupBy("foo")
+ }
+
+ @Test
+ def testGroupByKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work
+ customDs.groupBy("myInt")
+ }
+
+ @Test
+ def testGroupByKeyExpressions1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromCollection(customTypeData)
+
+ // should work
+ try {
+ ds.groupBy("myInt")
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyExpressions2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+
+ // should not work: groups on basic type
+ val longDs = env.fromCollection(emptyLongData)
+ longDs.groupBy("l")
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testGroupByKeyExpressions3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work: groups on custom type
+ customDs.groupBy(0)
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupByKeyExpressions4(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromCollection(customTypeData)
+
+ // should not work, non-existent field
+ ds.groupBy("myNonExistent")
+ }
+
+ @Test
+ def testGroupByKeySelector1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ try {
+ val customDs = env.fromCollection(customTypeData)
+ customDs.groupBy { _.myLong }
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test
+ def testGroupSortKeyFields1(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+ try {
+ tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testGroupSortKeyFields2(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+
+ // should not work, field position out of range
+ tupleDs.groupBy(0).sortGroup(5, Order.ASCENDING)
+ }
+
+ @Test(expected = classOf[InvalidProgramException])
+ def testGroupSortKeyFields3(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val longDs = env.fromCollection(emptyLongData)
+ longDs.groupBy { x: Long => x } .sortGroup(0, Order.ASCENDING)
+ }
+
+ @Test
+ def testChainedGroupSortKeyFields(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tupleDs = env.fromCollection(emptyTupleData)
+ try {
+ tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING).sortGroup(2, Order.DESCENDING)
+ }
+ catch {
+ case e: Exception => Assert.fail()
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
new file mode 100644
index 0000000..2605830
--- /dev/null
+++ b/flink-tests/src/test/scala/org/apache/flink/api/scala/operators/JoinITCase.scala
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.common.functions.RichJoinFunction
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object JoinProgs {
+ var NUM_PROGRAMS: Int = 20
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * UDF Join on tuples with key field positions
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
+
+ case 2 =>
+ /*
+ * UDF Join on tuples with multiple key field positions
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.get3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(0, 1).equalTo(0, 4) { (l, r) => (l._3, r._4) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt wie gehts?\n" +
+ "Hello world,ABC\n" + "I am fine.,HIJ\n" + "I am fine.,IJK\n"
+
+ case 3 =>
+ /*
+ * Default Join on tuples
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(0).equalTo(2)
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" + "(2,2,Hello),(2,3,2,Hallo Welt wie," +
+ "1)\n" + "(3,2,Hello world),(3,4,3,Hallo Welt wie gehts?,2)\n"
+
+ case 4 =>
+ /*
+ * Join with Huge
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.joinWithHuge(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
+
+ case 5 =>
+ /*
+ * Join with Tiny
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.joinWithTiny(ds2).where(1).equalTo(1) { (l, r) => (l._3, r._4) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt\n"
+
+ case 6 =>
+ /*
+ * Join that returns the left input object
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => l }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n"
+
+ case 7 =>
+ /*
+ * Join that returns the right input object
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(1).equalTo(1) { (l, r) => r }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt,2\n" + "2,2,1,Hallo Welt,2\n"
+
+ case 8 =>
+ /*
+ * Join with broadcast set
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds1 = CollectionDataSets.get3TupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(1).equalTo(4).apply(
+ new RichJoinFunction[
+ (Int, Long, String),
+ (Int, Long, Int, String, Long),
+ (String, String, Int)] {
+ private var broadcast = 41
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ broadcast = ints.sum
+ }
+
+ override def join(
+ first: (Int, Long, String),
+ second: (Int, Long, Int, String, Long)): (String, String, Int) = {
+ (first._3, second. _4, broadcast)
+ }
+ }
+ ).withBroadcastSet(intDs, "ints")
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo,55\n" + "Hi,Hallo Welt wie,55\n" + "Hello,Hallo Welt," +
+ "55\n" + "Hello world,Hallo Welt,55\n"
+
+ case 9 =>
+ /*
+ * Join on a tuple input with key field selector and a custom type input with key extractor
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallCustomTypeDataSet(env)
+ val ds2 = CollectionDataSets.get3TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where( _.myInt ).equalTo(0) { (l, r) => (l.myString, r._3) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hi\n" + "Hello,Hello\n" + "Hello world,Hello\n"
+
+ case 10 => // 12 in Java ITCase
+ /*
+ * Join on a tuple input with key field selector and a custom type input with key extractor
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmall3TupleDataSet(env)
+ val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
+ val joinDs = ds1.join(ds2).where(1).equalTo(_.myLong) apply { (l, r) => (l._3, r.myString) }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hello\n" + "Hello,Hello world\n" + "Hello world,Hello world\n"
+
+ case 11 => // 13 in Java ITCase
+ /*
+ * (Default) Join on two custom type inputs with key extractors
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getCustomTypeDataSet(env)
+ val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
+ val joinDs = ds1.join(ds2).where(_.myInt).equalTo(_.myInt)
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,0,Hi,1,0,Hi\n" + "2,1,Hello,2,1,Hello\n" + "2,1,Hello,2,2,Hello world\n" + "2," +
+ "2,Hello world,2,1,Hello\n" + "2,2,Hello world,2,2,Hello world\n"
+
+ case 12 => // 14 in Java ITCase
+ /*
+ * UDF Join on tuples with tuple-returning key selectors
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.get3TupleDataSet(env)
+ val ds2 = CollectionDataSets.get5TupleDataSet(env)
+ val joinDs = ds1.join(ds2).where( t => (t._1, t._2)).equalTo( t => (t._1, t._5)) apply {
+ (l, r) => (l._3, r._4)
+ }
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "Hi,Hallo\n" + "Hello,Hallo Welt\n" + "Hello world,Hallo Welt wie gehts?\n" +
+ "Hello world,ABC\n" + "I am fine.,HIJ\n" + "I am fine.,IJK\n"
+
+ /**
+ * Joins with POJOs
+ */
+ case 13 => // 15 in Java ITCase
+ /*
+ * Join nested pojo against tuple (selected using a string)
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val joinDs = ds1.join(ds2).where("nestedPojo.longNumber").equalTo("_7")
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
+ "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
+ "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
+
+ case 14 => // 16 in Java ITCase
+ /*
+ * Join nested pojo against tuple (selected as an integer)
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val joinDs = ds1.join(ds2).where("nestedPojo.longNumber").equalTo(6) // <-- difference
+ joinDs.writeAsCsv(resultPath)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
+ "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
+ "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
+
+ case 15 => // 17 in Java ITCase
+ /*
+ * selecting multiple fields using expression language
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val joinDs = ds1.join(ds2)
+ .where("nestedPojo.longNumber", "number", "str")
+ .equalTo("_7", "_1", "_2")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
+ "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
+ "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
+
+ case 16 => // 18 in Java ITCase
+ /*
+ * nested into tuple
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val joinDs = ds1.join(ds2).where("nestedPojo.longNumber", "number",
+ "nestedTupleWithCustom._1").equalTo("_7", "_1", "_3")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
+ "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
+ "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
+
+ case 17 => // 19 in Java ITCase
+ /*
+ * nested into tuple into pojo
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
+ val joinDs = ds1.join(ds2)
+ .where("nestedTupleWithCustom._1",
+ "nestedTupleWithCustom._2.myInt",
+ "nestedTupleWithCustom._2.myLong")
+ .equalTo("_3", "_4", "_5")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One," +
+ "10000)\n" + "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two," +
+ "20000)\n" + "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n"
+
+ case 18 => // 20 in Java ITCase
+ /*
+ * Non-POJO test to verify that full-tuple keys are working.
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env)
+ val joinDs = ds1.join(ds2).where(0).equalTo("_1._1", "_1._2")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "((1,1),one),((1,1),one)\n" + "((2,2),two),((2,2),two)\n" + "((3,3),three),((3,3)," +
+ "three)\n"
+
+ case 19 => // 21 in Java ITCase
+ /*
+ * Non-POJO test to verify "nested" tuple-element selection.
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env)
+ val ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env)
+ val joinDs = ds1.join(ds2).where("_1._1").equalTo("_1._1")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "((1,1),one),((1,1),one)\n" + "((2,2),two),((2,2),two)\n" + "((3,3),three),((3,3),three)\n"
+
+ case 20 => // 22 in Java ITCase
+ /*
+ * full pojo with full tuple
+ */
+ val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ val ds1 = CollectionDataSets.getSmallPojoDataSet(env)
+ val ds2 = CollectionDataSets.getSmallTuplebasedDataSetMatchingPojo(env)
+ val joinDs = ds1.join(ds2).where("*").equalTo("*")
+ joinDs.writeAsCsv(resultPath)
+ env.setDegreeOfParallelism(1)
+ env.execute()
+ "1 First (10,100,1000,One) 10000,(10000,10,100,1000,One,1,First)\n" +
+ "2 Second (20,200,2000,Two) 20000,(20000,20,200,2000,Two,2,Second)\n" +
+ "3 Third (30,300,3000,Three) 30000,(30000,30,300,3000,Three,3,Third)\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id: " + progId)
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class JoinITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = JoinProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object JoinITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to JoinProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
[9/9] git commit: [FLINK-1171] Move Scala API tests to flink-tests
project
Posted by rm...@apache.org.
[FLINK-1171] Move Scala API tests to flink-tests project
This closes #159
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/a0ad9031
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/a0ad9031
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/a0ad9031
Branch: refs/heads/master
Commit: a0ad90314f456f51e7889a3d0a77c279edf39f2d
Parents: ed1827c
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Oct 17 16:46:45 2014 +0200
Committer: Robert Metzger <me...@web.de>
Committed: Sat Oct 18 17:16:28 2014 +0200
----------------------------------------------------------------------
flink-scala/pom.xml | 23 -
.../scala/DeltaIterationSanityCheckTest.scala | 166 ----
.../api/scala/ScalaAPICompletenessTest.scala | 170 -----
.../SemanticPropertiesTranslationTest.scala | 194 -----
.../scala/io/CollectionInputFormatTest.scala | 133 ----
.../flink/api/scala/io/CsvInputFormatTest.scala | 288 -------
.../api/scala/operators/AggregateITCase.scala | 141 ----
.../scala/operators/AggregateOperatorTest.scala | 133 ----
.../api/scala/operators/CoGroupITCase.scala | 402 ----------
.../scala/operators/CoGroupOperatorTest.scala | 273 -------
.../flink/api/scala/operators/CrossITCase.scala | 239 ------
.../api/scala/operators/DistinctITCase.scala | 191 -----
.../scala/operators/DistinctOperatorTest.scala | 154 ----
.../api/scala/operators/ExamplesITCase.scala | 164 ----
.../api/scala/operators/FilterITCase.scala | 173 -----
.../api/scala/operators/FirstNITCase.scala | 116 ---
.../scala/operators/FirstNOperatorTest.scala | 168 -----
.../api/scala/operators/FlatMapITCase.scala | 219 ------
.../api/scala/operators/GroupReduceITCase.scala | 748 -------------------
.../api/scala/operators/GroupingTest.scala | 228 ------
.../flink/api/scala/operators/JoinITCase.scala | 376 ----------
.../api/scala/operators/JoinOperatorTest.scala | 276 -------
.../flink/api/scala/operators/MapITCase.scala | 242 ------
.../api/scala/operators/PartitionITCase.scala | 218 ------
.../api/scala/operators/ReduceITCase.scala | 236 ------
.../api/scala/operators/SumMinMaxITCase.scala | 146 ----
.../flink/api/scala/operators/UnionITCase.scala | 126 ----
.../translation/AggregateTranslationTest.scala | 61 --
.../DeltaIterationTranslationTest.scala | 254 -------
.../translation/DistinctTranslationTest.scala | 51 --
.../translation/ReduceTranslationTest.scala | 138 ----
.../runtime/GenericPairComparatorTest.scala | 76 --
.../scala/runtime/TupleComparatorILD2Test.scala | 56 --
.../scala/runtime/TupleComparatorILD3Test.scala | 54 --
.../runtime/TupleComparatorILDC3Test.scala | 54 --
.../runtime/TupleComparatorILDX1Test.scala | 54 --
.../runtime/TupleComparatorILDXC2Test.scala | 54 --
.../scala/runtime/TupleComparatorISD1Test.scala | 53 --
.../scala/runtime/TupleComparatorISD2Test.scala | 53 --
.../scala/runtime/TupleComparatorISD3Test.scala | 53 --
.../api/scala/runtime/TupleSerializerTest.scala | 190 -----
.../runtime/TupleSerializerTestInstance.scala | 90 ---
.../tuple/base/PairComparatorTestBase.scala | 102 ---
.../tuple/base/TupleComparatorTestBase.scala | 31 -
.../scala/types/TypeInformationGenTest.scala | 261 -------
.../api/scala/util/CollectionDataSets.scala | 426 -----------
flink-tests/pom.xml | 170 ++++-
.../scala/DeltaIterationSanityCheckTest.scala | 166 ++++
.../api/scala/ScalaAPICompletenessTest.scala | 170 +++++
.../SemanticPropertiesTranslationTest.scala | 194 +++++
.../scala/io/CollectionInputFormatTest.scala | 133 ++++
.../flink/api/scala/io/CsvInputFormatTest.scala | 288 +++++++
.../api/scala/operators/AggregateITCase.scala | 141 ++++
.../scala/operators/AggregateOperatorTest.scala | 133 ++++
.../api/scala/operators/CoGroupITCase.scala | 402 ++++++++++
.../scala/operators/CoGroupOperatorTest.scala | 273 +++++++
.../flink/api/scala/operators/CrossITCase.scala | 239 ++++++
.../api/scala/operators/DistinctITCase.scala | 191 +++++
.../scala/operators/DistinctOperatorTest.scala | 154 ++++
.../api/scala/operators/ExamplesITCase.scala | 164 ++++
.../api/scala/operators/FilterITCase.scala | 173 +++++
.../api/scala/operators/FirstNITCase.scala | 116 +++
.../scala/operators/FirstNOperatorTest.scala | 168 +++++
.../api/scala/operators/FlatMapITCase.scala | 219 ++++++
.../api/scala/operators/GroupReduceITCase.scala | 748 +++++++++++++++++++
.../api/scala/operators/GroupingTest.scala | 228 ++++++
.../flink/api/scala/operators/JoinITCase.scala | 376 ++++++++++
.../api/scala/operators/JoinOperatorTest.scala | 276 +++++++
.../flink/api/scala/operators/MapITCase.scala | 242 ++++++
.../api/scala/operators/PartitionITCase.scala | 218 ++++++
.../api/scala/operators/ReduceITCase.scala | 236 ++++++
.../api/scala/operators/SumMinMaxITCase.scala | 146 ++++
.../flink/api/scala/operators/UnionITCase.scala | 126 ++++
.../translation/AggregateTranslationTest.scala | 61 ++
.../DeltaIterationTranslationTest.scala | 254 +++++++
.../translation/DistinctTranslationTest.scala | 51 ++
.../translation/ReduceTranslationTest.scala | 138 ++++
.../runtime/GenericPairComparatorTest.scala | 76 ++
.../scala/runtime/TupleComparatorILD2Test.scala | 56 ++
.../scala/runtime/TupleComparatorILD3Test.scala | 54 ++
.../runtime/TupleComparatorILDC3Test.scala | 54 ++
.../runtime/TupleComparatorILDX1Test.scala | 54 ++
.../runtime/TupleComparatorILDXC2Test.scala | 54 ++
.../scala/runtime/TupleComparatorISD1Test.scala | 53 ++
.../scala/runtime/TupleComparatorISD2Test.scala | 53 ++
.../scala/runtime/TupleComparatorISD3Test.scala | 53 ++
.../api/scala/runtime/TupleSerializerTest.scala | 190 +++++
.../runtime/TupleSerializerTestInstance.scala | 90 +++
.../tuple/base/PairComparatorTestBase.scala | 102 +++
.../tuple/base/TupleComparatorTestBase.scala | 31 +
.../scala/types/TypeInformationGenTest.scala | 261 +++++++
.../api/scala/util/CollectionDataSets.scala | 426 +++++++++++
92 files changed, 8194 insertions(+), 8061 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index 827c7f0..4878cba 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -41,22 +41,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${project.version}</version>
</dependency>
@@ -100,13 +84,6 @@ under the License.
</dependency>
<dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.10</artifactId>
- <version>2.2.0</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
deleted file mode 100644
index 729fda4..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/DeltaIterationSanityCheckTest.scala
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala
-
-import org.junit.Test
-import org.apache.flink.api.common.InvalidProgramException
-
-// Verify that the sanity checking in delta iterations works. We just
-// have a dummy job that is not meant to be executed. Only verify that
-// the join/coGroup inside the iteration is checked.
-class DeltaIterationSanityCheckTest extends Serializable {
-
- @Test
- def testCorrectJoinWithSolution1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = s.join(ws).where("_1").equalTo("_1") { (l, r) => l }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test
- def testCorrectJoinWithSolution2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = ws.join(s).where("_1").equalTo("_1") { (l, r) => l }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectJoinWithSolution1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = s.join(ws).where("_2").equalTo("_2") { (l, r) => l }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectJoinWithSolution2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = ws.join(s).where("_2").equalTo("_2") { (l, r) => l }
- (result, ws)
- }
-
- iteration.print() }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectJoinWithSolution3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_2")) { (s, ws) =>
- val result = ws.join(s).where("_1").equalTo("_1") { (l, r) => l }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test
- def testCorrectCoGroupWithSolution1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = s.coGroup(ws).where("_1").equalTo("_1") { (l, r) => l.min }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test
- def testCorrectCoGroupWithSolution2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = ws.coGroup(s).where("_1").equalTo("_1") { (l, r) => l.min }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectCoGroupWithSolution1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = s.coGroup(ws).where("_2").equalTo("_2") { (l, r) => l.min }
- (result, ws)
- }
-
- iteration.print()
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectCoGroupWithSolution2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_1")) { (s, ws) =>
- val result = ws.coGroup(s).where("_2").equalTo("_2") { (l, r) => l.min }
- (result, ws)
- }
-
- iteration.print() }
-
- @Test(expected = classOf[InvalidProgramException])
- def testIncorrectCoGroupWithSolution3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val solutionInput = env.fromElements((1, "1"))
- val worksetInput = env.fromElements((2, "2"))
-
- val iteration = solutionInput.iterateDelta(worksetInput, 10, Array("_2")) { (s, ws) =>
- val result = ws.coGroup(s).where("_1").equalTo("_1") { (l, r) => l.min }
- (result, ws)
- }
-
- iteration.print()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
deleted file mode 100644
index 1aad4d1..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala
-
-import java.lang.reflect.Method
-
-import org.apache.flink.api.java.operators._
-import org.apache.flink.api.java.{DataSet => JavaDataSet}
-
-import scala.language.existentials
-
-import org.junit.Assert._
-import org.junit.Test
-
-/**
- * This checks whether the Scala API is up to feature parity with the Java API. Right now is very
- * simple, it is only checked whether a method with the same name exists.
- *
- * When adding excluded methods to the lists you should give a good reason in a comment.
- *
- * Note: This is inspired by the JavaAPICompletenessChecker from Spark.
- */
-class ScalaAPICompletenessTest {
-
- private def isExcludedByName(method: Method): Boolean = {
- val name = method.getDeclaringClass.getName + "." + method.getName
- val excludedNames = Seq(
- // These are only used internally. Should be internal API but Java doesn't have
- // private[flink].
- "org.apache.flink.api.java.DataSet.getExecutionEnvironment",
- "org.apache.flink.api.java.DataSet.getType",
- "org.apache.flink.api.java.operators.Operator.getResultType",
- "org.apache.flink.api.java.operators.Operator.getName",
- "org.apache.flink.api.java.operators.Grouping.getDataSet",
- "org.apache.flink.api.java.operators.Grouping.getKeys",
- "org.apache.flink.api.java.operators.SingleInputOperator.getInput",
- "org.apache.flink.api.java.operators.SingleInputOperator.getInputType",
- "org.apache.flink.api.java.operators.TwoInputOperator.getInput1",
- "org.apache.flink.api.java.operators.TwoInputOperator.getInput2",
- "org.apache.flink.api.java.operators.TwoInputOperator.getInput1Type",
- "org.apache.flink.api.java.operators.TwoInputOperator.getInput2Type",
- "org.apache.flink.api.java.ExecutionEnvironment.localExecutionIsAllowed",
- "org.apache.flink.api.java.ExecutionEnvironment.setDefaultLocalParallelism",
-
- // This is really just a mapper, which in Scala can easily expressed as a map lambda
- "org.apache.flink.api.java.DataSet.writeAsFormattedText",
-
- // Exclude minBy and maxBy for now, since there is some discussion about our aggregator
- // semantics
- "org.apache.flink.api.java.DataSet.minBy",
- "org.apache.flink.api.java.DataSet.maxBy",
- "org.apache.flink.api.java.operators.UnsortedGrouping.minBy",
- "org.apache.flink.api.java.operators.UnsortedGrouping.maxBy"
-
- )
- val excludedPatterns = Seq(
- // We don't have project on tuples in the Scala API
- """^org\.apache\.flink\.api.java.*project""",
-
- // I don't want to have withParameters in the API since I consider Configuration to be
- // deprecated. But maybe thats just me ...
- """^org\.apache\.flink\.api.java.*withParameters""",
-
- // These are only used internally. Should be internal API but Java doesn't have
- // private[flink].
- """^org\.apache\.flink\.api.java.*getBroadcastSets""",
- """^org\.apache\.flink\.api.java.*setSemanticProperties""",
- """^org\.apache\.flink\.api.java.*getSemanticProperties""",
- """^org\.apache\.flink\.api.java.*getParameters""",
-
- // Commented out for now until we have a use case for this.
- """^org\.apache\.flink\.api.java.*runOperation""",
-
- // Object methods
- """^.*notify""",
- """^.*wait""",
- """^.*notifyAll""",
- """^.*equals""",
- """^.*toString""",
- """^.*getClass""",
- """^.*hashCode"""
- ).map(_.r)
- lazy val excludedByPattern =
- excludedPatterns.map(_.findFirstIn(name)).filter(_.isDefined).nonEmpty
- name.contains("$") || excludedNames.contains(name) || excludedByPattern
- }
-
- private def isExcludedByInterface(method: Method): Boolean = {
- val excludedInterfaces =
- Set("org.apache.spark.Logging", "org.apache.hadoop.mapreduce.HadoopMapReduceUtil")
- def toComparisionKey(method: Method) =
- (method.getReturnType, method.getName, method.getGenericReturnType)
- val interfaces = method.getDeclaringClass.getInterfaces.filter { i =>
- excludedInterfaces.contains(i.getName)
- }
- val excludedMethods = interfaces.flatMap(_.getMethods.map(toComparisionKey))
- excludedMethods.contains(toComparisionKey(method))
- }
-
- private def checkMethods(
- javaClassName: String,
- scalaClassName: String,
- javaClass: Class[_],
- scalaClass: Class[_]) {
- val javaMethods = javaClass.getMethods
- .filterNot(_.isAccessible)
- .filterNot(isExcludedByName)
- .filterNot(isExcludedByInterface)
- .map(m => m.getName).toSet
-
- val scalaMethods = scalaClass.getMethods
- .filterNot(_.isAccessible)
- .filterNot(isExcludedByName)
- .filterNot(isExcludedByInterface)
- .map(m => m.getName).toSet
-
- val missingMethods = javaMethods -- scalaMethods
-
- for (method <- missingMethods) {
- fail("Method " + method + " from " + javaClass + " is missing from " + scalaClassName + ".")
- }
- }
-
- @Test
- def testCompleteness(): Unit = {
- checkMethods("DataSet", "DataSet", classOf[JavaDataSet[_]], classOf[DataSet[_]])
-
- checkMethods(
- "ExecutionEnvironment", "ExecutionEnvironment",
- classOf[org.apache.flink.api.java.ExecutionEnvironment], classOf[ExecutionEnvironment])
-
- checkMethods("Operator", "DataSet", classOf[Operator[_, _]], classOf[DataSet[_]])
-
- checkMethods("UnsortedGrouping", "GroupedDataSet",
- classOf[UnsortedGrouping[_]], classOf[GroupedDataSet[_]])
-
- checkMethods("SortedGrouping", "GroupedDataSet",
- classOf[SortedGrouping[_]], classOf[GroupedDataSet[_]])
-
- checkMethods("AggregateOperator", "AggregateDataSet",
- classOf[AggregateOperator[_]], classOf[AggregateDataSet[_]])
-
- checkMethods("SingleInputOperator", "DataSet",
- classOf[SingleInputOperator[_, _, _]], classOf[DataSet[_]])
-
- checkMethods("TwoInputOperator", "DataSet",
- classOf[TwoInputOperator[_, _, _, _]], classOf[DataSet[_]])
-
- checkMethods("SingleInputUdfOperator", "DataSet",
- classOf[SingleInputUdfOperator[_, _, _]], classOf[DataSet[_]])
-
- checkMethods("TwoInputUdfOperator", "DataSet",
- classOf[TwoInputUdfOperator[_, _, _, _]], classOf[DataSet[_]])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
deleted file mode 100644
index 543c05a..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/functions/SemanticPropertiesTranslationTest.scala
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.functions
-
-import org.junit.Assert._
-import org.apache.flink.api.common.functions.RichJoinFunction
-import org.apache.flink.api.common.functions.RichMapFunction
-import org.apache.flink.api.common.operators.{GenericDataSinkBase, SingleInputSemanticProperties}
-import org.apache.flink.api.common.operators.base.{JoinOperatorBase, MapOperatorBase}
-import org.apache.flink.api.common.operators.util.FieldSet
-import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFields
-import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsFirst
-import org.apache.flink.api.java.functions.FunctionAnnotation.ConstantFieldsSecond
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-/**
- * This is a minimal test to verify that semantic annotations are evaluated against
- * the type information properly translated correctly to the common data flow API.
- *
- * This covers only the constant fields annotations currently !!!
- */
-class SemanticPropertiesTranslationTest {
- /**
- * A mapper that preserves all fields over a tuple data set.
- */
- @Test
- def translateUnaryFunctionAnnotationTuplesWildCard(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val input = env.fromElements((3L, "test", 42))
- input.map(new WildcardConstantMapper[(Long, String, Int)]).print()
-
- val plan = env.createProgramPlan()
-
- val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
-
- val mapper: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_, _, _]]
-
- val semantics: SingleInputSemanticProperties = mapper.getSemanticProperties
- val fw1: FieldSet = semantics.getForwardedField(0)
- val fw2: FieldSet = semantics.getForwardedField(1)
- val fw3: FieldSet = semantics.getForwardedField(2)
-
- assertNotNull(fw1)
- assertNotNull(fw2)
- assertNotNull(fw3)
- assertTrue(fw1.contains(0))
- assertTrue(fw2.contains(1))
- assertTrue(fw3.contains(2))
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-
- /**
- * A mapper that preserves fields 0, 1, 2 of a tuple data set.
- */
- @Test
- def translateUnaryFunctionAnnotationTuples(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val input = env.fromElements((3L, "test", 42))
- input.map(new IndividualConstantMapper[Long, String, Int]).print()
-
- val plan = env.createProgramPlan()
-
- val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
-
- val mapper: MapOperatorBase[_, _, _] = sink.getInput.asInstanceOf[MapOperatorBase[_, _, _]]
-
- val semantics: SingleInputSemanticProperties = mapper.getSemanticProperties
- val fw1: FieldSet = semantics.getForwardedField(0)
- val fw2: FieldSet = semantics.getForwardedField(1)
- val fw3: FieldSet = semantics.getForwardedField(2)
-
- assertNotNull(fw1)
- assertNotNull(fw2)
- assertNotNull(fw3)
- assertTrue(fw1.contains(0))
- assertTrue(fw2.contains(1))
- assertTrue(fw3.contains(2))
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-
- /**
- * A join that preserves tuple fields from both sides.
- */
- @Test
- def translateBinaryFunctionAnnotationTuples(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val input1 = env.fromElements((3L, "test"))
- val input2 = env.fromElements((3L, 3.1415))
-
- input1.join(input2).where(0).equalTo(0)(
- new ForwardingTupleJoin[Long, String, Long, Double]).print()
-
- val plan = env.createProgramPlan()
- val sink: GenericDataSinkBase[_] = plan.getDataSinks.iterator.next
-
- val join: JoinOperatorBase[_, _, _, _] =
- sink.getInput.asInstanceOf[JoinOperatorBase[_, _, _, _]]
-
- val semantics = join.getSemanticProperties
- val fw11: FieldSet = semantics.getForwardedField1(0)
- val fw12: FieldSet = semantics.getForwardedField1(1)
- val fw21: FieldSet = semantics.getForwardedField2(0)
- val fw22: FieldSet = semantics.getForwardedField2(1)
-
- assertNull(fw11)
- assertNull(fw21)
- assertNotNull(fw12)
- assertNotNull(fw22)
- assertTrue(fw12.contains(0))
- assertTrue(fw22.contains(1))
- }
- catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- fail("Exception in test: " + e.getMessage)
- }
- }
- }
-}
-
-
-@ConstantFields(Array("*"))
-class WildcardConstantMapper[T] extends RichMapFunction[T, T] {
- def map(value: T): T = {
- value
- }
-}
-
-@ConstantFields(Array("0->0;1->1;2->2"))
-class IndividualConstantMapper[X, Y, Z] extends RichMapFunction[(X, Y, Z), (X, Y, Z)] {
- def map(value: (X, Y, Z)): (X, Y, Z) = {
- value
- }
-}
-
-@ConstantFields(Array("0"))
-class ZeroConstantMapper[T] extends RichMapFunction[T, T] {
- def map(value: T): T = {
- value
- }
-}
-
-@ConstantFieldsFirst(Array("1 -> 0"))
-@ConstantFieldsSecond(Array("1 -> 1"))
-class ForwardingTupleJoin[A, B, C, D] extends RichJoinFunction[(A, B), (C, D), (B, D)] {
- def join(first: (A, B), second: (C, D)): (B, D) = {
- (first._2, second._2)
- }
-}
-
-@ConstantFieldsFirst(Array("0 -> 0"))
-@ConstantFieldsSecond(Array("0 -> 1"))
-class ForwardingBasicJoin[A, B] extends RichJoinFunction[A, B, (A, B)] {
- def join(first: A, second: B): (A, B) = {
- (first, second)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
deleted file mode 100644
index a81dc45..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CollectionInputFormatTest.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.io
-
-import org.apache.flink.api.java.io.CollectionInputFormat
-import org.junit.Assert.assertEquals
-import org.junit.Assert.assertNotNull
-import org.junit.Assert.assertTrue
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.core.io.GenericInputSplit
-import org.junit.Test
-import java.io.ByteArrayInputStream
-import java.io.ByteArrayOutputStream
-import java.io.ObjectInputStream
-import java.io.ObjectOutputStream
-import org.apache.flink.api.scala._
-import scala.collection.JavaConverters._
-
-class ElementType(val id: Int) {
- def this() {
- this(-1)
- }
-
- override def equals(obj: Any): Boolean = {
- if (obj != null && obj.isInstanceOf[ElementType]) {
- val et = obj.asInstanceOf[ElementType]
- et.id == this.id
- }
- else {
- false
- }
- }
-}
-
-class CollectionInputFormatTest {
-
- @Test
- def testSerializability(): Unit = {
-
- val inputCollection = Seq(new ElementType(1), new ElementType(2), new ElementType(3))
- val info = createTypeInformation[ElementType]
-
- val inputFormat: CollectionInputFormat[ElementType] = new
- CollectionInputFormat[ElementType](inputCollection.asJava, info.createSerializer())
-
- val buffer = new ByteArrayOutputStream
- val out = new ObjectOutputStream(buffer)
-
- out.writeObject(inputFormat)
-
- val in = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray))
- val serializationResult: AnyRef = in.readObject
-
- assertNotNull(serializationResult)
- assertTrue(serializationResult.isInstanceOf[CollectionInputFormat[_]])
-
- val result = serializationResult.asInstanceOf[CollectionInputFormat[ElementType]]
- val inputSplit = new GenericInputSplit
- inputFormat.open(inputSplit)
- result.open(inputSplit)
-
- while (!inputFormat.reachedEnd && !result.reachedEnd) {
- val expectedElement = inputFormat.nextRecord(null)
- val actualElement = result.nextRecord(null)
- assertEquals(expectedElement, actualElement)
- }
- }
-
- @Test
- def testSerializabilityStrings(): Unit = {
- val data = Seq("To bey or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,", "And by opposing end them?--To die," +
- "--to sleep,--", "No more; and by a sleep to say we end", "The heartache, " +
- "and the thousand natural shocks", "That flesh is heir to,--'tis a consummation",
- "Devoutly to be wish'd. To die,--to sleep;--", "To sleep! perchance to dream:--ay, " +
- "there's the rub;", "For in that sleep of death what dreams may come,",
- "When we have shuffled off this mortal coil,", "Must give us pause: there's the respect",
- "That makes calamity of so long life;", "For who would bear the whips and scorns of time,",
- "The oppressor's wrong, the proud man's contumely,", "The pangs of despis'd love, " +
- "the law's delay,", "The insolence of office, and the spurns",
- "That patient merit of the unworthy takes,", "When he himself might his quietus make",
- "With a bare bodkin? who would these fardels bear,", "To grunt and sweat under a weary " +
- "life,", "But that the dread of something after death,--", "The undiscover'd country, " +
- "from whose bourn", "No traveller returns,--puzzles the will,",
- "And makes us rather bear those ills we have", "Than fly to others that we know not of?",
- "Thus conscience does make cowards of us all;", "And thus the native hue of resolution",
- "Is sicklied o'er with the pale cast of thought;", "And enterprises of great pith and " +
- "moment,", "With this regard, their currents turn awry,", "And lose the name of action" +
- ".--Soft you now!", "The fair Ophelia!--Nymph, in thy orisons",
- "Be all my sins remember'd.")
-
- val inputFormat = new CollectionInputFormat[String](
- data.asJava,
- BasicTypeInfo.STRING_TYPE_INFO.createSerializer)
- val baos = new ByteArrayOutputStream
- val oos = new ObjectOutputStream(baos)
-
- oos.writeObject(inputFormat)
- oos.close()
-
- val bais = new ByteArrayInputStream(baos.toByteArray)
- val ois = new ObjectInputStream(bais)
- val result: AnyRef = ois.readObject
-
- assertTrue(result.isInstanceOf[CollectionInputFormat[_]])
- var i: Int = 0
- val in = result.asInstanceOf[CollectionInputFormat[String]]
- in.open(new GenericInputSplit)
-
- while (!in.reachedEnd) {
- assertEquals(data(i), in.nextRecord(""))
- i += 1
- }
- assertEquals(data.length, i)
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
deleted file mode 100644
index 9c90788..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/io/CsvInputFormatTest.scala
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.io
-
-import org.apache.flink.api.scala.operators.ScalaCsvInputFormat
-import org.junit.Assert.assertEquals
-import org.junit.Assert.assertNotNull
-import org.junit.Assert.assertNull
-import org.junit.Assert.assertTrue
-import org.junit.Assert.fail
-import java.io.File
-import java.io.FileOutputStream
-import java.io.FileWriter
-import java.io.OutputStreamWriter
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.core.fs.FileInputSplit
-import org.apache.flink.core.fs.Path
-import org.junit.Test
-import org.apache.flink.api.scala._
-
-class CsvInputFormatTest {
-
- private final val PATH: Path = new Path("an/ignored/file/")
- private final val FIRST_PART: String = "That is the first part"
- private final val SECOND_PART: String = "That is the second part"
-
- @Test
- def readStringFields():Unit = {
- try {
- val fileContent = "abc|def|ghijk\nabc||hhg\n|||"
- val split = createTempFile(fileContent)
- val format = new ScalaCsvInputFormat[(String, String, String)](
- PATH, createTypeInformation[(String, String, String)])
- format.setDelimiter("\n")
- format.setFieldDelimiter('|')
- val parameters = new Configuration
- format.configure(parameters)
- format.open(split)
- var result: (String, String, String) = null
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("abc", result._1)
- assertEquals("def", result._2)
- assertEquals("ghijk", result._3)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("abc", result._1)
- assertEquals("", result._2)
- assertEquals("hhg", result._3)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("", result._1)
- assertEquals("", result._2)
- assertEquals("", result._3)
- result = format.nextRecord(result)
- assertNull(result)
- assertTrue(format.reachedEnd)
- }
- catch {
- case ex: Exception => {
- ex.printStackTrace()
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
- }
-
- @Test
- def readStringFieldsWithTrailingDelimiters(): Unit = {
- try {
- val fileContent = "abc|def|ghijk\nabc||hhg\n|||\n"
- val split = createTempFile(fileContent)
- val format = new ScalaCsvInputFormat[(String, String, String)](
- PATH, createTypeInformation[(String, String, String)])
- format.setDelimiter("\n")
- format.setFieldDelimiter('|')
- val parameters = new Configuration
- format.configure(parameters)
- format.open(split)
- var result: (String, String, String) = null
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("abc", result._1)
- assertEquals("def", result._2)
- assertEquals("ghijk", result._3)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("abc", result._1)
- assertEquals("", result._2)
- assertEquals("hhg", result._3)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals("", result._1)
- assertEquals("", result._2)
- assertEquals("", result._3)
- result = format.nextRecord(result)
- assertNull(result)
- assertTrue(format.reachedEnd)
- }
- catch {
- case ex: Exception =>
- ex.printStackTrace()
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
-
- @Test
- def testIntegerFields(): Unit = {
- try {
- val fileContent = "111|222|333|444|555\n666|777|888|999|000|\n"
- val split = createTempFile(fileContent)
- val format = new ScalaCsvInputFormat[(Int, Int, Int, Int, Int)](
- PATH, createTypeInformation[(Int, Int, Int, Int, Int)])
- format.setFieldDelimiter('|')
- format.configure(new Configuration)
- format.open(split)
- var result: (Int, Int, Int, Int, Int) = null
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(111), result._1)
- assertEquals(Integer.valueOf(222), result._2)
- assertEquals(Integer.valueOf(333), result._3)
- assertEquals(Integer.valueOf(444), result._4)
- assertEquals(Integer.valueOf(555), result._5)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(666), result._1)
- assertEquals(Integer.valueOf(777), result._2)
- assertEquals(Integer.valueOf(888), result._3)
- assertEquals(Integer.valueOf(999), result._4)
- assertEquals(Integer.valueOf(000), result._5)
- result = format.nextRecord(result)
- assertNull(result)
- assertTrue(format.reachedEnd)
- }
- catch {
- case ex: Exception =>
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
-
- @Test
- def testReadFirstN(): Unit = {
- try {
- val fileContent = "111|222|333|444|555|\n666|777|888|999|000|\n"
- val split = createTempFile(fileContent)
- val format = new ScalaCsvInputFormat[(Int, Int)](PATH, createTypeInformation[(Int, Int)])
- format.setFieldDelimiter('|')
- format.configure(new Configuration)
- format.open(split)
- var result: (Int, Int) = null
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(111), result._1)
- assertEquals(Integer.valueOf(222), result._2)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(666), result._1)
- assertEquals(Integer.valueOf(777), result._2)
- result = format.nextRecord(result)
- assertNull(result)
- assertTrue(format.reachedEnd)
- }
- catch {
- case ex: Exception =>
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
-
- @Test
- def testReadSparseWithPositionSetter(): Unit = {
- try {
- val fileContent: String = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666" +
- "|555|444|333|222|111|"
- val split = createTempFile(fileContent)
- val format = new ScalaCsvInputFormat[(Int, Int, Int)](
- PATH,
- createTypeInformation[(Int, Int, Int)])
- format.setFieldDelimiter('|')
- format.setFields(Array(0, 3, 7), Array(classOf[Integer], classOf[Integer], classOf[Integer]))
- format.configure(new Configuration)
- format.open(split)
- var result: (Int, Int, Int) = null
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(111), result._1)
- assertEquals(Integer.valueOf(444), result._2)
- assertEquals(Integer.valueOf(888), result._3)
- result = format.nextRecord(result)
- assertNotNull(result)
- assertEquals(Integer.valueOf(000), result._1)
- assertEquals(Integer.valueOf(777), result._2)
- assertEquals(Integer.valueOf(333), result._3)
- result = format.nextRecord(result)
- assertNull(result)
- assertTrue(format.reachedEnd)
- }
- catch {
- case ex: Exception =>
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
-
- @Test
- def testReadSparseWithShuffledPositions(): Unit = {
- try {
- val format = new ScalaCsvInputFormat[(Int, Int, Int)](
- PATH,
- createTypeInformation[(Int, Int, Int)])
- format.setFieldDelimiter('|')
- try {
- format.setFields(Array(8, 1, 3), Array(classOf[Integer],classOf[Integer],classOf[Integer]))
- fail("Input sequence should have been rejected.")
- }
- catch {
- case e: IllegalArgumentException => // ignore
- }
- }
- catch {
- case ex: Exception =>
- fail("Test failed due to a " + ex.getClass.getName + ": " + ex.getMessage)
- }
- }
-
- private def createTempFile(content: String): FileInputSplit = {
- val tempFile = File.createTempFile("test_contents", "tmp")
- tempFile.deleteOnExit()
- val wrt = new FileWriter(tempFile)
- wrt.write(content)
- wrt.close()
- new FileInputSplit(0, new Path(tempFile.toURI.toString), 0,
- tempFile.length,Array[String]("localhost"))
- }
-
- @Test
- def testWindowsLineEndRemoval(): Unit = {
- this.testRemovingTrailingCR("\n", "\n")
- this.testRemovingTrailingCR("\r\n", "\r\n")
- this.testRemovingTrailingCR("\r\n", "\n")
- }
-
- private def testRemovingTrailingCR(lineBreakerInFile: String, lineBreakerSetup: String) {
- var tempFile: File = null
- val fileContent = FIRST_PART + lineBreakerInFile + SECOND_PART + lineBreakerInFile
- try {
- tempFile = File.createTempFile("CsvInputFormatTest", "tmp")
- tempFile.deleteOnExit()
- tempFile.setWritable(true)
- val wrt = new OutputStreamWriter(new FileOutputStream(tempFile))
- wrt.write(fileContent)
- wrt.close()
- val inputFormat = new ScalaCsvInputFormat[Tuple1[String]](new Path(tempFile.toURI.toString),
- createTypeInformation[Tuple1[String]])
- val parameters = new Configuration
- inputFormat.configure(parameters)
- inputFormat.setDelimiter(lineBreakerSetup)
- val splits = inputFormat.createInputSplits(1)
- inputFormat.open(splits(0))
- var result = inputFormat.nextRecord(null)
- assertNotNull("Expecting to not return null", result)
- assertEquals(FIRST_PART, result._1)
- result = inputFormat.nextRecord(result)
- assertNotNull("Expecting to not return null", result)
- assertEquals(SECOND_PART, result._1)
- }
- catch {
- case t: Throwable =>
- System.err.println("test failed with exception: " + t.getMessage)
- t.printStackTrace(System.err)
- fail("Test erroneous")
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
deleted file mode 100644
index ad22fa6..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.java.aggregation.Aggregations
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object AggregateProgs {
- var NUM_PROGRAMS: Int = 3
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- // Full aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(10)
-// val ds = CollectionDataSets.get3TupleDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .aggregate(Aggregations.SUM,0)
- .and(Aggregations.MAX, 1)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map{ t => (t._1, t._2) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "231,6\n"
-
- case 2 =>
- // Grouped aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .groupBy(1)
- .aggregate(Aggregations.SUM, 0)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map { t => (t._2, t._1) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "1,1\n" + "2,5\n" + "3,15\n" + "4,34\n" + "5,65\n" + "6,111\n"
-
- case 3 =>
- // Nested aggregate
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
-
- val aggregateDs = ds
- .groupBy(1)
- .aggregate(Aggregations.MIN, 0)
- .aggregate(Aggregations.MIN, 0)
- // Ensure aggregate operator correctly copies other fields
- .filter(_._3 != null)
- .map { t => new Tuple1(t._1) }
-
- aggregateDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "1\n"
-
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class AggregateITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = AggregateProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object AggregateITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to AggregateProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
deleted file mode 100644
index 639884b..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.junit.Assert
-import org.apache.flink.api.common.InvalidProgramException
-import org.apache.flink.api.java.aggregation.Aggregations
-import org.apache.flink.api.java.aggregation.UnsupportedAggregationTypeException
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class AggregateOperatorTest {
-
- private final val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
- private final val tupleTypeInfo = createTypeInformation[(Int, Long, String, Long, Int)]
- private final val emptyLongData = Array[Long]()
-
- @Test
- def testFieldsAggregate(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- tupleDs.aggregate(Aggregations.SUM, 1)
- } catch {
- case e: Exception => Assert.fail()
- }
-
- // should not work: index out of bounds
- try {
- tupleDs.aggregate(Aggregations.SUM, 10)
- Assert.fail()
- } catch {
- case iae: IllegalArgumentException =>
- case e: Exception => Assert.fail()
- }
-
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work: not applied to tuple DataSet
- try {
- longDs.aggregate(Aggregations.MIN, 1)
- Assert.fail()
- } catch {
- case uoe: InvalidProgramException =>
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testFieldNamesAggregate(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work
- try {
- tupleDs.aggregate(Aggregations.SUM, "_2")
- } catch {
- case e: Exception => Assert.fail()
- }
-
- // should not work: invalid field
- try {
- tupleDs.aggregate(Aggregations.SUM, "foo")
- Assert.fail()
- } catch {
- case iae: IllegalArgumentException =>
- case e: Exception => Assert.fail()
- }
-
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work: not applied to tuple DataSet
- try {
- longDs.aggregate(Aggregations.MIN, "_1")
- Assert.fail()
- } catch {
- case uoe: InvalidProgramException =>
- case uoe: UnsupportedOperationException =>
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testAggregationTypes(): Unit = {
- try {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work: multiple aggregates
- tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4)
-
- // should work: nested aggregates
- tupleDs.aggregate(Aggregations.MIN, 2).and(Aggregations.SUM, 1)
-
- // should not work: average on string
- try {
- tupleDs.aggregate(Aggregations.SUM, 2)
- Assert.fail()
- } catch {
- case iae: UnsupportedAggregationTypeException =>
- }
- } catch {
- case e: Exception => {
- System.err.println(e.getMessage)
- e.printStackTrace()
- Assert.fail(e.getMessage)
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
deleted file mode 100644
index d4ce3b7..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupITCase.scala
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichCoGroupFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.apache.flink.util.Collector
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-import org.junit.Assert
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object CoGroupProgs {
- var NUM_PROGRAMS: Int = 13
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * CoGroup on tuples with key field selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
- (first, second) =>
- var sum = 0
- var id = 0
- for (t <- first) {
- sum += t._3
- id = t._1
- }
- for (t <- second) {
- sum += t._3
- id = t._1
- }
- (id, sum)
- }
- coGroupDs.writeAsCsv(resultPath)
- env.execute()
- "1,0\n" + "2,6\n" + "3,24\n" + "4,60\n" + "5,120\n"
-
- case 2 =>
- /*
- * CoGroup on two custom type inputs with key extractors
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
-
- val coGroupDs = ds.coGroup(ds2).where(_.myInt).equalTo(_.myInt) apply {
- (first, second) =>
- val o = new CustomType(0, 0, "test")
- for (c <- first) {
- o.myInt = c.myInt
- o.myLong += c.myLong
- }
- for (c <- second) {
- o.myInt = c.myInt
- o.myLong += c.myLong
- }
- o
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "1,0,test\n" + "2,6,test\n" + "3,24,test\n" + "4,60,test\n" + "5,120,test\n" + "6," +
- "210,test\n"
-
- case 3 =>
- /*
- * check correctness of cogroup if UDF returns left input objects
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val ds2 = CollectionDataSets.get3TupleDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
- (first, second, out: Collector[(Int, Long, String)] ) =>
- for (t <- first) {
- if (t._1 < 6) {
- out.collect(t)
- }
- }
- }
- coGroupDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
- "how are you?\n" + "5,3,I am fine.\n"
-
- case 4 =>
- /*
- * check correctness of cogroup if UDF returns right input objects
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0) {
- (first, second, out: Collector[(Int, Long, Int, String, Long)]) =>
- for (t <- second) {
- if (t._1 < 4) {
- out.collect(t)
- }
- }
- }
- coGroupDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt,2\n" + "2,3,2,Hallo Welt wie,1\n" + "3,4,3," +
- "Hallo Welt wie gehts?,2\n" + "3,5,4,ABC,2\n" + "3,6,5,BCD,3\n"
-
- case 5 =>
- val env = ExecutionEnvironment.getExecutionEnvironment
- val intDs = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.get5TupleDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).apply(
- new RichCoGroupFunction[
- (Int, Long, Int, String, Long),
- (Int, Long, Int, String, Long),
- (Int, Int, Int)] {
- private var broadcast = 41
-
- override def open(config: Configuration) {
- val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- broadcast = ints.sum
- }
-
- override def coGroup(
- first: java.lang.Iterable[(Int, Long, Int, String, Long)],
- second: java.lang.Iterable[(Int, Long, Int, String, Long)],
- out: Collector[(Int, Int, Int)]): Unit = {
- var sum = 0
- var id = 0
- for (t <- first.asScala) {
- sum += t._3
- id = t._1
- }
- for (t <- second.asScala) {
- sum += t._3
- id = t._1
- }
- out.collect((id, sum, broadcast))
- }
-
- }).withBroadcastSet(intDs, "ints")
- coGroupDs.writeAsCsv(resultPath)
- env.execute()
- "1,0,55\n" + "2,6,55\n" + "3,24,55\n" + "4,60,55\n" + "5,120,55\n"
-
- case 6 =>
- /*
- * CoGroup on a tuple input with key field selector and a custom type input with
- * key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(2).equalTo(_.myInt) apply {
- (first, second) =>
- var sum = 0L
- var id = 0
- for (t <- first) {
- sum += t._1
- id = t._3
- }
- for (t <- second) {
- sum += t.myLong
- id = t.myInt
- }
- (id, sum, "test")
- }
- coGroupDs.writeAsCsv(resultPath)
- env.execute()
- "0,1,test\n" + "1,2,test\n" + "2,5,test\n" + "3,15,test\n" + "4,33,test\n" + "5," +
- "63,test\n" + "6,109,test\n" + "7,4,test\n" + "8,4,test\n" + "9,4,test\n" + "10,5," +
- "test\n" + "11,5,test\n" + "12,5,test\n" + "13,5,test\n" + "14,5,test\n"
-
- case 7 =>
- /*
- * CoGroup on a tuple input with key field selector and a custom type input with
- * key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
- val coGroupDs = ds2.coGroup(ds).where(_.myInt).equalTo(2) {
- (first, second) =>
- var sum = 0L
- var id = 0
- for (t <- first) {
- sum += t.myLong
- id = t.myInt
- }
- for (t <- second) {
- sum += t._1
- id = t._3
- }
-
- new CustomType(id, sum, "test")
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "0,1,test\n" + "1,2,test\n" + "2,5,test\n" + "3,15,test\n" + "4,33,test\n" + "5," +
- "63,test\n" + "6,109,test\n" + "7,4,test\n" + "8,4,test\n" + "9,4,test\n" + "10,5," +
- "test\n" + "11,5,test\n" + "12,5,test\n" + "13,5,test\n" + "14,5,test\n"
-
- case 8 =>
- /*
- * CoGroup with multiple key fields
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets.get5TupleDataSet(env)
- val ds2 = CollectionDataSets.get3TupleDataSet(env)
- val coGrouped = ds1.coGroup(ds2).where(0,4).equalTo(0, 1) {
- (first, second, out: Collector[(Int, Long, String)]) =>
- val strs = first map(_._4)
- for (t <- second) {
- for (s <- strs) {
- out.collect((t._1, t._2, s))
- }
- }
- }
-
- coGrouped.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hallo\n" + "2,2,Hallo Welt\n" + "3,2,Hallo Welt wie gehts?\n" + "3,2," +
- "ABC\n" + "5,3,HIJ\n" + "5,3,IJK\n"
-
- case 9 =>
- /*
- * CoGroup with multiple key fields
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = CollectionDataSets
- .get5TupleDataSet(env)
- val ds2 = CollectionDataSets.get3TupleDataSet(env)
- val coGrouped = ds1.coGroup(ds2).where(t => (t._1, t._5)).equalTo(t => (t._1, t._2))
- .apply {
- (first, second, out: Collector[(Int, Long, String)]) =>
- val strs = first map(_._4)
- for (t <- second) {
- for (s <- strs) {
- out.collect((t._1, t._2, s))
- }
- }
- }
-
- coGrouped.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hallo\n" + "2,2,Hallo Welt\n" + "3,2,Hallo Welt wie gehts?\n" + "3,2," +
- "ABC\n" + "5,3,HIJ\n" + "5,3,IJK\n"
-
- case 10 =>
- /*
- * CoGroup on two custom type inputs using expression keys
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val ds2 = CollectionDataSets.getCustomTypeDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where("myInt").equalTo("myInt") {
- (first, second) =>
- val o = new CustomType(0, 0, "test")
- for (t <- first) {
- o.myInt = t.myInt
- o.myLong += t.myLong
- }
- for (t <- second) {
- o.myInt = t.myInt
- o.myLong += t.myLong
- }
- o
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "1,0,test\n" + "2,6,test\n" + "3,24,test\n" + "4,60,test\n" + "5,120,test\n" + "6," +
- "210,test\n"
-
- case 11 =>
- /*
- * CoGroup on two custom type inputs using expression keys
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where("nestedPojo.longNumber").equalTo(6) {
- (first, second, out: Collector[CustomType]) =>
- for (p <- first) {
- for (t <- second) {
- Assert.assertTrue(p.nestedPojo.longNumber == t._7)
- out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
- }
- }
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
-
- case 12 =>
- /*
- * CoGroup field-selector (expression keys) + key selector function
- * The key selector is unnecessary complicated (Tuple1) ;)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(t => new Tuple1(t.nestedPojo.longNumber)).equalTo(6) {
- (first, second, out: Collector[CustomType]) =>
- for (p <- first) {
- for (t <- second) {
- Assert.assertTrue(p.nestedPojo.longNumber == t._7)
- out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
- }
- }
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
-
- case 13 =>
- /*
- * CoGroup field-selector (expression keys) + key selector function
- * The key selector is simple here
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmallPojoDataSet(env)
- val ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env)
- val coGroupDs = ds.coGroup(ds2).where(_.nestedPojo.longNumber).equalTo(6) {
- (first, second, out: Collector[CustomType]) =>
- for (p <- first) {
- for (t <- second) {
- Assert.assertTrue(p.nestedPojo.longNumber == t._7)
- out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"))
- }
- }
- }
- coGroupDs.writeAsText(resultPath)
- env.execute()
- "-1,20000,Flink\n" + "-1,10000,Flink\n" + "-1,30000,Flink\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class CoGroupITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private val curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = CoGroupProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object CoGroupITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to CoGroupProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
deleted file mode 100644
index 3f0ca5f..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
-import org.junit.Assert
-import org.junit.Test
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-
-class CoGroupOperatorTest {
-
- private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
- private var customTypeData = Array[CustomType](new CustomType())
-
- @Test
- def testCoGroupKeyFields1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should work
- try {
- ds1.coGroup(ds2).where(0).equalTo(0)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyFields2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, incompatible key types
- ds1.coGroup(ds2).where(0).equalTo(2)
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyFields3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, incompatible number of key fields
- ds1.coGroup(ds2).where(0, 1).equalTo(2)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testCoGroupKeyFields4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, field position out of range
- ds1.coGroup(ds2).where(5).equalTo(0)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testCoGroupKeyFields5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, negative field position
- ds1.coGroup(ds2).where(-1).equalTo(-1)
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testCoGroupKeyFields6(): Unit = {
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should not work, field position key on custom data type
- ds1.coGroup(ds2).where(5).equalTo(0)
- }
-
- @Test
- def testCoGroupKeyFieldNames1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should work
- try {
- ds1.coGroup(ds2).where("_1").equalTo("_1")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyFieldNames2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, incompatible key types
- ds1.coGroup(ds2).where("_1").equalTo("_3")
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyFieldNames3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, incompatible number of key fields
- ds1.coGroup(ds2).where("_1", "_2").equalTo("_3")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testCoGroupKeyFieldNames4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, invalid field name
- ds1.coGroup(ds2).where("_6").equalTo("_1")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testCoGroupKeyFieldNames5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should not work, invalid field name
- ds1.coGroup(ds2).where("_1").equalTo("bar")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testCoGroupKeyFieldNames6(): Unit = {
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should not work, field position key on custom data type
- ds1.coGroup(ds2).where("_3").equalTo("_1")
- }
-
- @Test
- def testCoGroupKeyExpressions1(): Unit = {
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should work
- try {
-// ds1.coGroup(ds2).where("i").equalTo("i");
-
- }catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyExpressions2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, incompatible key types
- ds1.coGroup(ds2).where("myInt").equalTo("myString")
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyExpressions3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // should not work, incompatible number of keys
- ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString")
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testCoGroupKeyExpressions4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
-
- // should not work, key non-existent
- ds1.coGroup(ds2).where("myNonExistent").equalTo("i")
- }
-
- @Test
- def testCoGroupKeySelectors1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should work
- try {
- ds1.coGroup(ds2).where { _.myLong } equalTo { _.myLong }
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testCoGroupKeyMixing1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(customTypeData)
- val ds2 = env.fromCollection(emptyTupleData)
-
- // Should work
- try {
- ds1.coGroup(ds2).where { _.myLong }.equalTo(3)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test
- def testCoGroupKeyMixing2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should work
- try {
- ds1.coGroup(ds2).where(3).equalTo { _.myLong }
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyMixing3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should not work, incompatible types
- ds1.coGroup(ds2).where(2).equalTo { _.myLong }
- }
-
- @Test(expected = classOf[IncompatibleKeysException])
- def testCoGroupKeyMixing4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds1 = env.fromCollection(emptyTupleData)
- val ds2 = env.fromCollection(customTypeData)
-
- // Should not work, more than one field position key
- ds1.coGroup(ds2).where(1, 3).equalTo { _.myLong }
- }
-}
-
-
[8/9] [FLINK-1171] Move Scala API tests to flink-tests project
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
deleted file mode 100644
index bcb85e5..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CrossITCase.scala
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichCrossFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object CrossProgs {
- var NUM_PROGRAMS: Int = 9
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * check correctness of cross on two tuple inputs
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.cross(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4) }
- crossDs.writeAsCsv(resultPath)
- env.execute()
-
- "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
- "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
- "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
- "Hallo Welt wieHallo Welt wie\n"
-
- case 2 =>
- /*
- * check correctness of cross if UDF returns left input object
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.cross(ds2) { (l, r ) => l }
- crossDs.writeAsCsv(resultPath)
- env.execute()
-
- "1,1,Hi\n" + "1,1,Hi\n" + "1,1,Hi\n" + "2,2,Hello\n" + "2,2,Hello\n" + "2,2," +
- "Hello\n" + "3,2,Hello world\n" + "3,2,Hello world\n" + "3,2,Hello world\n"
-
- case 3 =>
- /*
- * check correctness of cross if UDF returns right input object
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.cross(ds2) { (l, r) => r }
- crossDs.writeAsCsv(resultPath)
- env.execute()
-
- "1,1,0,Hallo,1\n" + "1,1,0,Hallo,1\n" + "1,1,0,Hallo,1\n" + "2,2,1,Hallo Welt," +
- "2\n" + "2,2,1,Hallo Welt,2\n" + "2,2,1,Hallo Welt,2\n" + "2,3,2,Hallo Welt wie," +
- "1\n" + "2,3,2,Hallo Welt wie,1\n" + "2,3,2,Hallo Welt wie,1\n"
-
- case 4 =>
- /*
- * check correctness of cross with broadcast set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val intDs = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.cross(ds2).apply (
- new RichCrossFunction[
- (Int, Long, Int, String, Long),
- (Int, Long, Int, String, Long),
- (Int, Int, Int)] {
- private var broadcast = 41
-
- override def open(config: Configuration) {
- val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- broadcast = ints.sum
- }
-
- override def cross(
- first: (Int, Long, Int, String, Long),
- second: (Int, Long, Int, String, Long)): (Int, Int, Int) = {
- (first._1 + second._1, first._3.toInt * second._3.toInt, broadcast)
- }
-
- })withBroadcastSet(intDs, "ints")
- crossDs.writeAsCsv(resultPath)
- env.execute()
- "2,0,55\n" + "3,0,55\n" + "3,0,55\n" + "3,0,55\n" + "4,1,55\n" + "4,2,55\n" + "3," +
- "0,55\n" + "4,2,55\n" + "4,4,55\n"
-
- case 5 =>
- /*
- * check correctness of crossWithHuge (only correctness of result -> should be the same
- * as with normal cross)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.crossWithHuge(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4)}
- crossDs.writeAsCsv(resultPath)
- env.execute()
- "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
- "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
- "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
- "Hallo Welt wieHallo Welt wie\n"
-
- case 6 =>
- /*
- * check correctness of crossWithTiny (only correctness of result -> should be the same
- * as with normal cross)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets
- .getSmall5TupleDataSet(env)
- val ds2 = CollectionDataSets
- .getSmall5TupleDataSet(env)
- val crossDs = ds.crossWithTiny(ds2) { (l, r) => (l._3 + r._3, l._4 + r._4)}
- crossDs.writeAsCsv(resultPath)
- env.execute()
- "0,HalloHallo\n" + "1,HalloHallo Welt\n" + "2,HalloHallo Welt wie\n" + "1," +
- "Hallo WeltHallo\n" + "2,Hallo WeltHallo Welt\n" + "3,Hallo WeltHallo Welt wie\n" + "2," +
- "Hallo Welt wieHallo\n" + "3,Hallo Welt wieHallo Welt\n" + "4," +
- "Hallo Welt wieHallo Welt wie\n"
-
- case 7 => // 9 in Java CrossITCase
- /*
- * check correctness of default cross
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmall5TupleDataSet(env)
- val crossDs = ds.cross(ds2)
- crossDs.writeAsCsv(resultPath)
- env.execute()
- "(1,1,Hi),(2,2,1,Hallo Welt,2)\n" + "(1,1,Hi),(1,1,0,Hallo,1)\n" + "(1,1,Hi),(2,3," +
- "2,Hallo Welt wie,1)\n" + "(2,2,Hello),(2,2,1,Hallo Welt,2)\n" + "(2,2,Hello),(1,1,0," +
- "Hallo,1)\n" + "(2,2,Hello),(2,3,2,Hallo Welt wie,1)\n" + "(3,2,Hello world),(2,2,1," +
- "Hallo Welt,2)\n" + "(3,2,Hello world),(1,1,0,Hallo,1)\n" + "(3,2,Hello world),(2,3,2," +
- "Hallo Welt wie,1)\n"
-
- case 8 => // 10 in Java CrossITCase
- /*
- * check correctness of cross on two custom type inputs
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmallCustomTypeDataSet(env)
- val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
- val crossDs = ds.cross(ds2) {
- (l, r) => new CustomType(l.myInt * r.myInt, l.myLong + r.myLong, l.myString + r.myString)
- }
- crossDs.writeAsText(resultPath)
- env.execute()
- "1,0,HiHi\n" + "2,1,HiHello\n" + "2,2,HiHello world\n" + "2,1,HelloHi\n" + "4,2," +
- "HelloHello\n" + "4,3,HelloHello world\n" + "2,2,Hello worldHi\n" + "4,3," +
- "Hello worldHello\n" + "4,4,Hello worldHello world"
-
- case 9 => // 11 in Java CrossITCase
- /*
- * check correctness of cross a tuple input and a custom type input
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
- val ds2 = CollectionDataSets.getSmallCustomTypeDataSet(env)
- val crossDs = ds.cross(ds2) {
- (l, r) => (l._1 + r.myInt, l._3 * r.myLong, l._4 + r.myString)
- }
- crossDs.writeAsCsv(resultPath)
- env.execute()
- "2,0,HalloHi\n" + "3,0,HalloHello\n" + "3,0,HalloHello world\n" + "3,0," +
- "Hallo WeltHi\n" + "4,1,Hallo WeltHello\n" + "4,2,Hallo WeltHello world\n" + "3,0," +
- "Hallo Welt wieHi\n" + "4,2,Hallo Welt wieHello\n" + "4,4,Hallo Welt wieHello world\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class CrossITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = CrossProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object CrossITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to CrossProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
deleted file mode 100644
index 855335d..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object DistinctProgs {
- var NUM_PROGRAMS: Int = 8
-
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Check correctness of distinct on tuples with key field selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
-
- val distinctDs = ds.union(ds).distinct(0, 1, 2)
- distinctDs.writeAsCsv(resultPath)
-
- env.execute()
-
- // return expected result
- "1,1,Hi\n" +
- "2,2,Hello\n" +
- "3,2,Hello world\n"
-
- case 2 =>
- /*
- * check correctness of distinct on tuples with key field selector with not all fields
- * selected
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
-
- val distinctDs = ds.union(ds).distinct(0).map(_._1)
-
- distinctDs.writeAsText(resultPath)
- env.execute()
- "1\n" + "2\n"
-
- case 3 =>
- /*
- * check correctness of distinct on tuples with key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
-
- val reduceDs = ds.union(ds).distinct(_._1).map(_._1)
-
- reduceDs.writeAsText(resultPath)
- env.execute()
- "1\n" + "2\n"
-
- case 4 =>
- /*
- * check correctness of distinct on custom type with type extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
-
- val reduceDs = ds.distinct(_.myInt).map( t => new Tuple1(t.myInt))
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
-
- case 5 =>
- /*
- * check correctness of distinct on tuples
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall3TupleDataSet(env)
-
- val distinctDs = ds.union(ds).distinct()
-
- distinctDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n"
-
- case 6 =>
- /*
- * check correctness of distinct on custom type with tuple-returning type extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get5TupleDataSet(env)
-
- val reduceDs = ds.distinct( t => (t._1, t._5)).map( t => (t._1, t._5) )
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1\n" + "2,1\n" + "2,2\n" + "3,2\n" + "3,3\n" + "4,1\n" + "4,2\n" + "5," +
- "1\n" + "5,2\n" + "5,3\n"
-
- case 7 =>
- /*
- * check correctness of distinct on tuples with field expressions
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getSmall5TupleDataSet(env)
-
- val reduceDs = ds.union(ds).distinct("_1").map(t => new Tuple1(t._1))
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1\n" + "2\n"
-
- case 8 =>
- /*
- * check correctness of distinct on Pojos
- */
-
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
-
- val reduceDs = ds.distinct("nestedPojo.longNumber").map(_.nestedPojo.longNumber.toInt)
-
- reduceDs.writeAsText(resultPath)
- env.execute()
- "10000\n20000\n30000\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class DistinctITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = DistinctProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object DistinctITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to DistinctProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
deleted file mode 100644
index e9d214b..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
-import org.junit.Assert
-import org.apache.flink.api.common.InvalidProgramException
-import org.junit.Test
-
-import org.apache.flink.api.scala._
-
-class DistinctOperatorTest {
-
- private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
- private val customTypeData = Array[CustomType](new CustomType())
- private val emptyLongData = Array[Long]()
-
- @Test
- def testDistinctByKeyIndices1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // Should work
- try {
- tupleDs.distinct(0)
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testDistinctByKeyIndices2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work: distinct on basic type
- longDs.distinct(0)
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testDistinctByKeyIndices3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work: field position key on custom type
- customDs.distinct(0)
- }
-
- @Test
- def testDistinctByKeyIndices4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should work
- tupleDs.distinct()
- }
-
- @Test(expected = classOf[InvalidProgramException])
- def testDistinctByKeyIndices5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work: distinct on custom type
- customDs.distinct()
- }
-
- @Test(expected = classOf[IllegalArgumentException])
- def testDistinctByKeyIndices6(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, negative field position key
- tupleDs.distinct(-1)
- }
-
- @Test
- def testDistinctByKeyFields1(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // Should work
- try {
- tupleDs.distinct("_1")
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-
- @Test(expected = classOf[RuntimeException])
- def testDistinctByKeyFields2(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val longDs = env.fromCollection(emptyLongData)
-
- // should not work: distinct on basic type
- longDs.distinct("_1")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testDistinctByKeyFields3(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should not work: invalid fields
- customDs.distinct("_1")
- }
-
- @Test(expected = classOf[RuntimeException])
- def testDistinctByKeyFields4(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- // should not work, invalid field
- tupleDs.distinct("foo")
- }
-
- @Test
- def testDistinctByKeyFields5(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val customDs = env.fromCollection(customTypeData)
-
- // should work
- customDs.distinct("myInt")
- }
-
- @Test
- def testDistinctByKeySelector1(): Unit = {
- val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
- try {
- val customDs = env.fromCollection(customTypeData)
- customDs.distinct {_.myLong}
- }
- catch {
- case e: Exception => Assert.fail()
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
deleted file mode 100644
index f43052a..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.apache.flink.api.scala._
-import org.junit.runners.Parameterized.Parameters
-import scala.collection.JavaConverters._
-
-import scala.collection.mutable
-
-// TODO case class Tuple2[T1, T2](_1: T1, _2: T2)
-// TODO case class Foo(a: Int, b: String
-
-case class Nested(myLong: Long)
-
-class Pojo(var myString: String, var myInt: Int, var nested: Nested) {
- def this() = {
- this("", 0, new Nested(1))
- }
-
- def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
-
- override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
-}
-
-class NestedPojo(var myLong: Long) {
- def this() { this(0) }
-}
-
-class PojoWithPojo(var myString: String, var myInt: Int, var nested: Nested) {
- def this() = {
- this("", 0, new Nested(1))
- }
-
- def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
-
- override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
-}
-
-object ExampleProgs {
- var NUM_PROGRAMS: Int = 4
-
- def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
- progId match {
- case 1 =>
- /*
- Test nested tuples with int offset
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
-
- val grouped = ds.groupBy(0).reduce( { (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)})
- grouped.writeAsText(resultPath)
- env.execute()
- "((this,hello),3)\n((this,is),3)\n"
-
- case 2 =>
- /*
- Test nested tuples with int offset
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
-
- val grouped = ds.groupBy("_1._1").reduce{
- (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)
- }
- grouped.writeAsText(resultPath)
- env.execute()
- "((this,is),6)\n"
-
- case 3 =>
- /*
- Test nested pojos
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements(
- new PojoWithPojo("one", 1, 1L),
- new PojoWithPojo("one", 1, 1L),
- new PojoWithPojo("two", 666, 2L) )
-
- val grouped = ds.groupBy("nested.myLong").reduce {
- (p1, p2) =>
- p1.myInt += p2.myInt
- p1
- }
- grouped.writeAsText(resultPath)
- env.execute()
- "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
-
- case 4 =>
- /*
- Test pojo with nested case class
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements(
- new Pojo("one", 1, 1L),
- new Pojo("one", 1, 1L),
- new Pojo("two", 666, 2L) )
-
- val grouped = ds.groupBy("nested.myLong").reduce {
- (p1, p2) =>
- p1.myInt += p2.myInt
- p1
- }
- grouped.writeAsText(resultPath)
- env.execute()
- "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
- }
- }
-}
-
-@RunWith(classOf[Parameterized])
-class ExamplesITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = ExampleProgs.runProgram(curProgId, resultPath, isCollectionExecution)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object ExamplesITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to ExampleProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
deleted file mode 100644
index 973028b..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FilterITCase.scala
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichFilterFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object FilterProgs {
- var NUM_PROGRAMS: Int = 7
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Test all-rejecting filter.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val filterDs = ds.filter( t => false )
- filterDs.writeAsCsv(resultPath)
- env.execute()
- "\n"
-
- case 2 =>
- /*
- * Test all-passing filter.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val filterDs = ds.filter( t => true )
- filterDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, " +
- "how are you?\n" + "5,3,I am fine.\n" + "6,3,Luke Skywalker\n" + "7,4," +
- "Comment#1\n" + "8,4,Comment#2\n" + "9,4,Comment#3\n" + "10,4,Comment#4\n" + "11,5," +
- "Comment#5\n" + "12,5,Comment#6\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "15,5," +
- "Comment#9\n" + "16,6,Comment#10\n" + "17,6,Comment#11\n" + "18,6,Comment#12\n" + "19," +
- "6,Comment#13\n" + "20,6,Comment#14\n" + "21,6,Comment#15\n"
-
- case 3 =>
- /*
- * Test filter on String tuple field.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val filterDs = ds.filter( _._3.contains("world") )
- filterDs.writeAsCsv(resultPath)
- env.execute()
- "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
-
- case 4 =>
- /*
- * Test filter on Integer tuple field.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val filterDs = ds.filter( _._1 % 2 == 0 )
- filterDs.writeAsCsv(resultPath)
- env.execute()
- "2,2,Hello\n" + "4,3,Hello world, how are you?\n" + "6,3,Luke Skywalker\n" + "8,4," +
- "Comment#2\n" + "10,4,Comment#4\n" + "12,5,Comment#6\n" + "14,5,Comment#8\n" + "16,6," +
- "Comment#10\n" + "18,6,Comment#12\n" + "20,6,Comment#14\n"
-
- case 5 =>
- /*
- * Test filter on basic type
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getStringDataSet(env)
- val filterDs = ds.filter( _.startsWith("H") )
- filterDs.writeAsText(resultPath)
- env.execute()
- "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n"
-
- case 6 =>
- /*
- * Test filter on custom type
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val filterDs = ds.filter( _.myString.contains("a") )
- filterDs.writeAsText(resultPath)
- env.execute()
- "3,3,Hello world, how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n"
-
- case 7 =>
- /*
- * Test filter on String tuple field.
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ints = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val filterDs = ds.filter( new RichFilterFunction[(Int, Long, String)] {
- var literal = -1
- override def open(config: Configuration): Unit = {
- val ints = getRuntimeContext.getBroadcastVariable[Int]("ints")
- for (i <- ints.asScala) {
- literal = if (literal < i) i else literal
- }
- }
- override def filter(value: (Int, Long, String)): Boolean = value._1 < literal
- }).withBroadcastSet(ints, "ints")
- filterDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n" + "4,3,Hello world, how are you?\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class FilterITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = FilterProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object FilterITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to FilterProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
deleted file mode 100644
index 6882885..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNITCase.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object FirstNProgs {
- var NUM_PROGRAMS: Int = 3
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * First-n on ungrouped data set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val seven = ds.first(7).map( t => new Tuple1(1) ).sum(0)
- seven.writeAsText(resultPath)
- env.execute()
- "(7)\n"
-
- case 2 =>
- /*
- * First-n on grouped data set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val first = ds.groupBy(1).first(4).map( t => (t._2, 1)).groupBy(0).sum(1)
- first.writeAsText(resultPath)
- env.execute()
- "(1,1)\n(2,2)\n(3,3)\n(4,4)\n(5,4)\n(6,4)\n"
-
- case 3 =>
- /*
- * First-n on grouped and sorted data set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val first = ds.groupBy(1)
- .sortGroup(0, Order.DESCENDING)
- .first(3)
- .map ( t => (t._2, t._1))
- first.writeAsText(resultPath)
- env.execute()
- "(1,1)\n" + "(2,3)\n(2,2)\n" + "(3,6)\n(3,5)\n(3,4)\n" + "(4,10)\n(4,9)\n(4," +
- "8)\n" + "(5,15)\n(5,14)\n(5,13)\n" + "(6,21)\n(6,20)\n(6,19)\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class FirstNITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = FirstNProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object FirstNITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to FirstNProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
deleted file mode 100644
index 7c259b8..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FirstNOperatorTest.scala
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.InvalidProgramException
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.junit.{Assert, Test}
-
-import org.apache.flink.api.scala._
-
-class FirstNOperatorTest {
-
- private val emptyTupleData = Array[(Int, Long, String, Long, Int)]()
-
- @Test
- def testUngroupedFirstN(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- val tupleDs = env.fromCollection(emptyTupleData)
-
- try {
- tupleDs.first(1)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.first(10)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.first(0)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.first(-1)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- }
-
- @Test
- def testGroupedFirstN(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- try {
- tupleDs.groupBy(2).first(1)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(1, 3).first(10)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(0).first(0)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(2).first(-1)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- }
-
- @Test
- def testGroupedSortedFirstN(): Unit = {
- val env = ExecutionEnvironment.getExecutionEnvironment
- val tupleDs = env.fromCollection(emptyTupleData)
-
- try {
- tupleDs.groupBy(2).sortGroup(4, Order.ASCENDING).first(1)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(1, 3).sortGroup(4, Order.ASCENDING).first(10)
- }
- catch {
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(0).sortGroup(4, Order.ASCENDING).first(0)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- try {
- tupleDs.groupBy(2).sortGroup(4, Order.ASCENDING).first(-1)
- Assert.fail()
- }
- catch {
- case ipe: InvalidProgramException => {
- }
- case e: Exception => {
- Assert.fail()
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
deleted file mode 100644
index 0d80d22..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/FlatMapITCase.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.MutableTuple3
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.apache.flink.util.Collector
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object FlatMapProgs {
- var NUM_PROGRAMS: Int = 7
-
- def runProgram(progId: Int, resultPath: String): String = {
- progId match {
- case 1 =>
- /*
- * Test non-passing flatmap
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getStringDataSet(env)
- val nonPassingFlatMapDs = ds.flatMap( in => if (in.contains("banana")) Some(in) else None )
- nonPassingFlatMapDs.writeAsText(resultPath)
- env.execute()
- "\n"
-
- case 2 =>
- /*
- * Test data duplicating flatmap
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getStringDataSet(env)
- val duplicatingFlatMapDs = ds.flatMap( in => Seq(in, in.toUpperCase) )
- duplicatingFlatMapDs.writeAsText(resultPath)
- env.execute()
- "Hi\n" + "HI\n" + "Hello\n" + "HELLO\n" + "Hello world\n" + "HELLO WORLD\n" +
- "Hello world, how are you?\n" + "HELLO WORLD, HOW ARE YOU?\n" + "I am fine.\n" + "I AM " +
- "FINE.\n" + "Luke Skywalker\n" + "LUKE SKYWALKER\n" + "Random comment\n" + "RANDOM " +
- "COMMENT\n" + "LOL\n" + "LOL\n"
-
- case 3 =>
- /*
- * Test flatmap with varying number of emitted tuples
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val varyingTuplesMapDs = ds.flatMap {
- in =>
- val numTuples = in._1 % 3
- (0 until numTuples) map { i => in }
- }
- varyingTuplesMapDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "2,2,Hello\n" + "4,3,Hello world, " +
- "how are you?\n" + "5,3,I am fine.\n" + "5,3,I am fine.\n" + "7,4,Comment#1\n" + "8,4," +
- "Comment#2\n" + "8,4,Comment#2\n" + "10,4,Comment#4\n" + "11,5,Comment#5\n" + "11,5," +
- "Comment#5\n" + "13,5,Comment#7\n" + "14,5,Comment#8\n" + "14,5,Comment#8\n" + "16,6," +
- "Comment#10\n" + "17,6,Comment#11\n" + "17,6,Comment#11\n" + "19,6,Comment#13\n" + "20," +
- "6,Comment#14\n" + "20,6,Comment#14\n"
-
- case 4 =>
- /*
- * Test type conversion flatmapper (Custom -> Tuple)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val typeConversionFlatMapDs = ds.flatMap { in => Some((in.myInt, in.myLong, in.myString)) }
- typeConversionFlatMapDs.writeAsCsv(resultPath)
- env.execute()
- "1,0,Hi\n" + "2,1,Hello\n" + "2,2,Hello world\n" + "3,3,Hello world, " +
- "how are you?\n" + "3,4,I am fine.\n" + "3,5,Luke Skywalker\n" + "4,6," +
- "Comment#1\n" + "4,7,Comment#2\n" + "4,8,Comment#3\n" + "4,9,Comment#4\n" + "5,10," +
- "Comment#5\n" + "5,11,Comment#6\n" + "5,12,Comment#7\n" + "5,13,Comment#8\n" + "5,14," +
- "Comment#9\n" + "6,15,Comment#10\n" + "6,16,Comment#11\n" + "6,17,Comment#12\n" + "6," +
- "18,Comment#13\n" + "6,19,Comment#14\n" + "6,20,Comment#15\n"
-
- case 5 =>
- /*
- * Test type conversion flatmapper (Tuple -> Basic)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val typeConversionFlatMapDs = ds.flatMap ( in => Some(in._3) )
- typeConversionFlatMapDs.writeAsText(resultPath)
- env.execute()
- "Hi\n" + "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "I am fine" +
- ".\n" + "Luke Skywalker\n" + "Comment#1\n" + "Comment#2\n" + "Comment#3\n" +
- "Comment#4\n" + "Comment#5\n" + "Comment#6\n" + "Comment#7\n" + "Comment#8\n" +
- "Comment#9\n" + "Comment#10\n" + "Comment#11\n" + "Comment#12\n" + "Comment#13\n" +
- "Comment#14\n" + "Comment#15\n"
-
- case 6 =>
- /*
- * Test flatmapper if UDF returns input object
- * multiple times and changes it in between
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).map {
- t => MutableTuple3(t._1, t._2, t._3)
- }
- val inputObjFlatMapDs = ds.flatMap {
- (in, out: Collector[MutableTuple3[Int, Long, String]]) =>
- val numTuples = in._1 % 4
- (0 until numTuples) foreach { i => in._1 = i; out.collect(in) }
- }
- inputObjFlatMapDs.writeAsCsv(resultPath)
- env.execute()
- "0,1,Hi\n" + "0,2,Hello\n" + "1,2,Hello\n" + "0,2,Hello world\n" + "1,2," +
- "Hello world\n" + "2,2,Hello world\n" + "0,3,I am fine.\n" + "0,3," +
- "Luke Skywalker\n" + "1,3,Luke Skywalker\n" + "0,4,Comment#1\n" + "1,4," +
- "Comment#1\n" + "2,4,Comment#1\n" + "0,4,Comment#3\n" + "0,4,Comment#4\n" + "1,4," +
- "Comment#4\n" + "0,5,Comment#5\n" + "1,5,Comment#5\n" + "2,5,Comment#5\n" + "0,5," +
- "Comment#7\n" + "0,5,Comment#8\n" + "1,5,Comment#8\n" + "0,5,Comment#9\n" + "1,5," +
- "Comment#9\n" + "2,5,Comment#9\n" + "0,6,Comment#11\n" + "0,6,Comment#12\n" + "1,6," +
- "Comment#12\n" + "0,6,Comment#13\n" + "1,6,Comment#13\n" + "2,6,Comment#13\n" + "0,6," +
- "Comment#15\n"
-
- case 7 =>
- /*
- * Test flatmap with broadcast set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ints = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env).map {
- t => MutableTuple3(t._1, t._2, t._3)
- }
- val bcFlatMapDs = ds.flatMap(
- new RichFlatMapFunction[MutableTuple3[Int, Long, String],
- MutableTuple3[Int, Long, String]] {
- private var f2Replace = 0
- private val outTuple = MutableTuple3(0, 0L, "")
- override def open(config: Configuration): Unit = {
- val ints = getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- f2Replace = ints.sum
- }
- override def flatMap(
- value: MutableTuple3[Int, Long, String],
- out: Collector[MutableTuple3[Int, Long, String]]): Unit = {
- outTuple._1 = f2Replace
- outTuple._2 = value._2
- outTuple._3 = value._3
- out.collect(outTuple)
- }
- }).withBroadcastSet(ints, "ints")
- bcFlatMapDs.writeAsCsv(resultPath)
- env.execute()
- "55,1,Hi\n" + "55,2,Hello\n" + "55,2,Hello world\n" + "55,3,Hello world, " +
- "how are you?\n" + "55,3,I am fine.\n" + "55,3,Luke Skywalker\n" + "55,4," +
- "Comment#1\n" + "55,4,Comment#2\n" + "55,4,Comment#3\n" + "55,4,Comment#4\n" + "55,5," +
- "Comment#5\n" + "55,5,Comment#6\n" + "55,5,Comment#7\n" + "55,5,Comment#8\n" + "55,5," +
- "Comment#9\n" + "55,6,Comment#10\n" + "55,6,Comment#11\n" + "55,6,Comment#12\n" + "55," +
- "6,Comment#13\n" + "55,6,Comment#14\n" + "55,6,Comment#15\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class FlatMapITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private var curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = FlatMapProgs.runProgram(curProgId, resultPath)
- }
-
- protected override def postSubmit(): Unit = {
- compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object FlatMapITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to FlatMapProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a0ad9031/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
deleted file mode 100644
index 7b81933..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
+++ /dev/null
@@ -1,748 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import java.lang.Iterable
-
-import org.apache.flink.api.common.functions._
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.api.scala.ExecutionEnvironment
-import org.apache.flink.api.scala.util.CollectionDataSets
-import org.apache.flink.api.scala.util.CollectionDataSets.{CrazyNested, POJO, MutableTuple3,
-CustomType}
-import org.apache.flink.compiler.PactCompiler
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.test.util.JavaProgramTestBase
-import org.apache.flink.util.Collector
-import org.junit.runner.RunWith
-import org.junit.runners.Parameterized
-import org.junit.runners.Parameterized.Parameters
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.flink.api.scala._
-
-
-object GroupReduceProgs {
- var NUM_PROGRAMS: Int = 26
-
- def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
- progId match {
- case 1 =>
- /*
- * check correctness of groupReduce on tuples with key field selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1).reduceGroup {
- in =>
- in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
-
- case 2 =>
- /*
- * check correctness of groupReduce on tuples with multiple key field selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets
- .get5TupleDataSet(env)
- val reduceDs = ds.groupBy(4, 0).reduceGroup {
- in =>
- val (i, l, l2) = in
- .map( t => (t._1, t._2, t._5))
- .reduce((l, r) => (l._1, l._2 + r._2, l._3))
- (i, l, 0, "P-)", l2)
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
- "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
- "2\n" + "5,25,0,P-),3\n"
-
- case 3 =>
- /*
- * check correctness of groupReduce on tuples with key field selector and group sorting
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1).sortGroup(2, Order.ASCENDING).reduceGroup {
- in =>
- in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" +
- "5,2,Hello-Hello world\n" +
- "15,3,Hello world, how are you?-I am fine.-Luke Skywalker\n" +
- "34,4,Comment#1-Comment#2-Comment#3-Comment#4\n" +
- "65,5,Comment#5-Comment#6-Comment#7-Comment#8-Comment#9\n" +
- "111,6,Comment#10-Comment#11-Comment#12-Comment#13-Comment#14-Comment#15\n"
-
- case 4 =>
- /*
- * check correctness of groupReduce on tuples with key extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(_._2).reduceGroup {
- in =>
- in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
-
- case 5 =>
- /*
- * check correctness of groupReduce on custom type with type extractor
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val reduceDs = ds.groupBy(_.myInt).reduceGroup {
- in =>
- val iter = in.toIterator
- val o = new CustomType
- val c = iter.next()
-
- o.myString = "Hello!"
- o.myInt = c.myInt
- o.myLong = c.myLong
-
- while (iter.hasNext) {
- val next = iter.next()
- o.myLong += next.myLong
- }
- o
- }
- reduceDs.writeAsText(resultPath)
- env.execute()
- "1,0,Hello!\n" + "2,3,Hello!\n" + "3,12,Hello!\n" + "4,30,Hello!\n" + "5,60," +
- "Hello!\n" + "6,105,Hello!\n"
-
- case 6 =>
- /*
- * check correctness of all-groupreduce for tuples
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.reduceGroup {
- in =>
- var i = 0
- var l = 0L
- for (t <- in) {
- i += t._1
- l += t._2
- }
- (i, l, "Hello World")
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "231,91,Hello World\n"
-
- case 7 =>
- /*
- * check correctness of all-groupreduce for custom types
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
- val reduceDs = ds.reduceGroup {
- in =>
- val o = new CustomType(0, 0, "Hello!")
- for (t <- in) {
- o.myInt += t.myInt
- o.myLong += t.myLong
- }
- o
- }
- reduceDs.writeAsText(resultPath)
- env.execute()
- "91,210,Hello!"
-
- case 8 =>
- /*
- * check correctness of groupReduce with broadcast set
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val intDs = CollectionDataSets.getIntDataSet(env)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1).reduceGroup(
- new RichGroupReduceFunction[(Int, Long, String), (Int, Long, String)] {
- private var f2Replace = ""
-
- override def open(config: Configuration) {
- val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
- f2Replace = ints.sum + ""
- }
-
- override def reduce(
- values: Iterable[(Int, Long, String)],
- out: Collector[(Int, Long, String)]): Unit = {
- var i: Int = 0
- var l: Long = 0L
- for (t <- values.asScala) {
- i += t._1
- l = t._2
- }
- out.collect((i, l, f2Replace))
- }
- }).withBroadcastSet(intDs, "ints")
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,55\n" + "5,2,55\n" + "15,3,55\n" + "34,4,55\n" + "65,5,55\n" + "111,6,55\n"
-
- case 9 =>
- /*
- * check correctness of groupReduce if UDF returns input objects multiple times and
- * changes it in between
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env)
- .map( t => MutableTuple3(t._1, t._2, t._3) )
- val reduceDs = ds.groupBy(1).reduceGroup {
- (in, out: Collector[MutableTuple3[Int, Long, String]]) =>
- for (t <- in) {
- if (t._1 < 4) {
- t._3 = "Hi!"
- t._1 += 10
- out.collect(t)
- t._1 += 10
- t._3 = "Hi again!"
- out.collect(t)
- }
- }
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "11,1,Hi!\n" + "21,1,Hi again!\n" + "12,2,Hi!\n" + "22,2,Hi again!\n" + "13,2," +
- "Hi!\n" + "23,2,Hi again!\n"
-
- case 10 =>
- /*
- * check correctness of groupReduce on custom type with key extractor and combine
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getCustomTypeDataSet(env)
-
- @RichGroupReduceFunction.Combinable
- class CustomTypeGroupReduceWithCombine
- extends RichGroupReduceFunction[CustomType, CustomType] {
- override def combine(values: Iterable[CustomType], out: Collector[CustomType]): Unit = {
- val o = new CustomType()
- for (c <- values.asScala) {
- o.myInt = c.myInt
- o.myLong += c.myLong
- o.myString = "test" + c.myInt
- }
- out.collect(o)
- }
-
- override def reduce(values: Iterable[CustomType], out: Collector[CustomType]): Unit = {
- val o = new CustomType(0, 0, "")
- for (c <- values.asScala) {
- o.myInt = c.myInt
- o.myLong += c.myLong
- o.myString = c.myString
- }
- out.collect(o)
- }
- }
- val reduceDs = ds.groupBy(_.myInt).reduceGroup(new CustomTypeGroupReduceWithCombine)
-
- reduceDs.writeAsText(resultPath)
- env.execute()
- if (onCollection) {
- null
- }
- else {
- "1,0,test1\n" + "2,3,test2\n" + "3,12,test3\n" + "4,30,test4\n" + "5,60," +
- "test5\n" + "6,105,test6\n"
- }
-
- case 11 =>
- /*
- * check correctness of groupReduce on tuples with combine
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- // important because it determines how often the combiner is called
- env.setDegreeOfParallelism(2)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- @RichGroupReduceFunction.Combinable
- class Tuple3GroupReduceWithCombine
- extends RichGroupReduceFunction[(Int, Long, String), (Int, String)] {
- override def combine(
- values: Iterable[(Int, Long, String)],
- out: Collector[(Int, Long, String)]): Unit = {
- var i = 0
- var l = 0L
- var s = ""
- for (t <- values.asScala) {
- i += t._1
- l = t._2
- s = "test" + t._2
- }
- out.collect((i, l, s))
- }
-
- override def reduce(
- values: Iterable[(Int, Long, String)],
- out: Collector[(Int, String)]): Unit = {
- var i = 0
- var s = ""
- for (t <- values.asScala) {
- i += t._1
- s = t._3
- }
- out.collect((i, s))
- }
- }
- val reduceDs = ds.groupBy(1).reduceGroup(new Tuple3GroupReduceWithCombine)
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- if (onCollection) {
- null
- }
- else {
- "1,test1\n" + "5,test2\n" + "15,test3\n" + "34,test4\n" + "65,test5\n" + "111," +
- "test6\n"
- }
-
-
- // all-groupreduce with combine
-
-
- case 12 =>
- /*
- * check correctness of all-groupreduce for tuples with combine
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.get3TupleDataSet(env).map(t => t).setParallelism(4)
-
- val cfg: Configuration = new Configuration
- cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)
-
- @RichGroupReduceFunction.Combinable
- class Tuple3AllGroupReduceWithCombine
- extends RichGroupReduceFunction[(Int, Long, String), (Int, String)] {
- override def combine(
- values: Iterable[(Int, Long, String)],
- out: Collector[(Int, Long, String)]): Unit = {
- var i = 0
- var l = 0L
- var s = ""
- for (t <- values.asScala) {
- i += t._1
- l += t._2
- s += "test"
- }
- out.collect((i, l, s))
- }
-
- override def reduce(
- values: Iterable[(Int, Long, String)],
- out: Collector[(Int, String)]): Unit = {
- var i = 0
- var s = ""
- for (t <- values.asScala) {
- i += t._1 + t._2.toInt
- s += t._3
- }
- out.collect((i, s))
- }
- }
- val reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine).withParameters(cfg)
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- if (onCollection) {
- null
- }
- else {
- "322," +
- "testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest\n"
- }
-
- case 13 =>
- /*
- * check correctness of groupReduce with descending group sort
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1).sortGroup(2, Order.DESCENDING).reduceGroup {
- in =>
- in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
- }
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
- "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
- "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
- "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
-
- case 14 =>
- /*
- * check correctness of groupReduce on tuples with tuple-returning key selector
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets
- .get5TupleDataSet(env)
- val reduceDs = ds.groupBy( t => (t._1, t._5)).reduceGroup {
- in =>
- val (i, l, l2) = in
- .map( t => (t._1, t._2, t._5))
- .reduce((l, r) => (l._1, l._2 + r._2, l._3))
- (i, l, 0, "P-)", l2)
- }
-
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
- "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
- "2\n" + "5,25,0,P-),3\n"
-
- case 15 =>
- /*
- * check that input of combiner is also sorted for combinable groupReduce with group
- * sorting
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.get3TupleDataSet(env).map { t =>
- MutableTuple3(t._1, t._2, t._3)
- }
-
- @RichGroupReduceFunction.Combinable
- class OrderCheckingCombinableReduce
- extends RichGroupReduceFunction[MutableTuple3[Int, Long, String],
- MutableTuple3[Int, Long, String]] {
- def reduce(
- values: Iterable[MutableTuple3[Int, Long, String]],
- out: Collector[MutableTuple3[Int, Long, String]]) {
- val it = values.iterator()
- var t = it.next()
- val i = t._1
- out.collect(t)
-
- while (it.hasNext) {
- t = it.next()
- if (i > t._1 || (t._3 == "INVALID-ORDER!")) {
- t._3 = "INVALID-ORDER!"
- out.collect(t)
- }
- }
- }
-
- override def combine(
- values: Iterable[MutableTuple3[Int, Long, String]],
- out: Collector[MutableTuple3[Int, Long, String]]) {
- val it = values.iterator()
- var t = it.next
- val i: Int = t._1
- out.collect(t)
- while (it.hasNext) {
- t = it.next
- if (i > t._1) {
- t._3 = "INVALID-ORDER!"
- out.collect(t)
- }
- }
- }
- }
-
- val reduceDs = ds.groupBy(1)
- .sortGroup(0, Order.ASCENDING).reduceGroup(new OrderCheckingCombinableReduce)
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "2,2,Hello\n" + "4,3,Hello world, how are you?\n" + "7,4," +
- "Comment#1\n" + "11,5,Comment#5\n" + "16,6,Comment#10\n"
-
- case 16 =>
- /*
- * Deep nesting test
- * + null value in pojo
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets
- .getCrazyNestedDataSet(env)
- val reduceDs = ds.groupBy("nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal")
- .reduceGroup {
- in =>
- var c = 0
- var n: String = null
- for (v <- in) {
- c += 1
- n = v.nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal
- }
- (n, c)
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "aa,1\nbb,2\ncc,3\n"
-
-
- case 17 =>
- // We don't have that test but keep numbering compatible to Java GroupReduceITCase
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.fromElements("Hello world").writeAsText(resultPath)
- env.execute()
- "Hello world"
-
- case 18 =>
- /*
- * Test Pojo containing a Writable and Tuples
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets
- .getPojoContainingTupleAndWritable(env)
- val reduceDs = ds.groupBy("hadoopFan", "theTuple.*").reduceGroup(new
- GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, Integer] {
- def reduce(
- values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
- out: Collector[Integer]) {
- var c: Int = 0
- for (v <- values.asScala) {
- c += 1
- }
- out.collect(c)
- }
- })
- reduceDs.writeAsText(resultPath)
- env.execute()
- "1\n5\n"
-
- case 19 =>
- /*
- * Test Tuple containing pojos and regular fields
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = CollectionDataSets.getTupleContainingPojos(env)
- val reduceDs = ds.groupBy("_1", "_2.*").reduceGroup(
- new GroupReduceFunction[(Int, CrazyNested, POJO), Int] {
- def reduce(values: Iterable[(Int, CrazyNested, POJO)], out: Collector[Int]) {
- var c: Int = 0
- for (v <- values.asScala) {
- c += 1
- }
- out.collect(c)
- }
- })
- reduceDs.writeAsText(resultPath)
- env.execute()
- "3\n1\n"
-
- case 20 =>
- /*
- * Test string-based definition on group sort, based on test:
- * check correctness of groupReduce with descending group sort
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.get3TupleDataSet(env)
- val reduceDs = ds.groupBy(1)
- .sortGroup("_3", Order.DESCENDING)
- .reduceGroup {
- in =>
- in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
- }
- reduceDs.writeAsCsv(resultPath)
- env.execute()
- "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
- "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
- "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
- "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
-
- case 21 =>
- /*
- * Test int-based definition on group sort, for (full) nested Tuple
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
- val reduceDs = ds.groupBy("_2").sortGroup(0, Order.DESCENDING)
- .reduceGroup(new NestedTupleReducer)
- reduceDs.writeAsText(resultPath)
- env.execute()
- "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"
-
-
- case 22 =>
- /*
- * Test int-based definition on group sort, for (partial) nested Tuple ASC
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
- val reduceDs = ds.groupBy("_2")
- .sortGroup("_1._1", Order.ASCENDING)
- .sortGroup("_1._2", Order.ASCENDING)
- .reduceGroup(new NestedTupleReducer)
- reduceDs.writeAsText(resultPath)
- env.execute()
- "a--(1,2)-(1,3)-(2,1)-\n" + "b--(2,2)-\n" + "c--(3,3)-(3,6)-(4,9)-\n"
-
- case 23 =>
- /*
- * Test string-based definition on group sort, for (partial) nested Tuple DESC
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
- val reduceDs = ds.groupBy("_2")
- .sortGroup("_1._1", Order.DESCENDING)
- .sortGroup("_1._2", Order.ASCENDING)
- .reduceGroup(new NestedTupleReducer)
- reduceDs.writeAsText(resultPath)
- env.execute()
- "a--(2,1)-(1,2)-(1,3)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,3)-(3,6)-\n"
-
- case 24 =>
- /*
- * Test string-based definition on group sort, for two grouping keys
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env)
- val reduceDs = ds.groupBy("_2")
- .sortGroup("_1._1", Order.DESCENDING)
- .sortGroup("_1._2", Order.DESCENDING)
- .reduceGroup(new NestedTupleReducer)
- reduceDs.writeAsText(resultPath)
- env.execute()
- "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"
-
- case 25 =>
- /*
- * Test string-based definition on group sort, for two grouping keys with Pojos
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getGroupSortedPojoContainingTupleAndWritable(env)
- val reduceDs = ds.groupBy("hadoopFan")
- .sortGroup("theTuple._1", Order.DESCENDING)
- .sortGroup("theTuple._2", Order.DESCENDING)
- .reduceGroup(
- new GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, String] {
- def reduce(
- values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
- out: Collector[String]) {
- var once: Boolean = false
- val concat: StringBuilder = new StringBuilder
- for (value <- values.asScala) {
- if (!once) {
- concat.append(value.hadoopFan.get)
- concat.append("---")
- once = true
- }
- concat.append(value.theTuple)
- concat.append("-")
- }
- out.collect(concat.toString())
- }
- })
- reduceDs.writeAsText(resultPath)
- env.execute()
- "1---(10,100)-\n" + "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n"
-
- case 26 =>
- /*
- * Test grouping with pojo containing multiple pojos (was a bug)
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- env.setDegreeOfParallelism(1)
- val ds = CollectionDataSets.getPojoWithMultiplePojos(env)
- val reduceDs = ds.groupBy("p2.a2")
- .reduceGroup(
- new GroupReduceFunction[CollectionDataSets.PojoWithMultiplePojos, String] {
- def reduce(
- values: Iterable[CollectionDataSets.PojoWithMultiplePojos],
- out: Collector[String]) {
- val concat: StringBuilder = new StringBuilder
- for (value <- values.asScala) {
- concat.append(value.p2.a2)
- }
- out.collect(concat.toString())
- }
- })
- reduceDs.writeAsText(resultPath)
- env.execute()
- "b\nccc\nee\n"
-
- case _ =>
- throw new IllegalArgumentException("Invalid program id")
- }
- }
-}
-
-
-@RunWith(classOf[Parameterized])
-class GroupReduceITCase(config: Configuration) extends JavaProgramTestBase(config) {
-
- private val curProgId: Int = config.getInteger("ProgramId", -1)
- private var resultPath: String = null
- private var expectedResult: String = null
-
- protected override def preSubmit(): Unit = {
- resultPath = getTempDirPath("result")
- }
-
- protected def testProgram(): Unit = {
- expectedResult = GroupReduceProgs.runProgram(curProgId, resultPath, isCollectionExecution)
- }
-
- protected override def postSubmit(): Unit = {
- if (expectedResult != null) compareResultsByLinesInMemory(expectedResult, resultPath)
- }
-}
-
-object GroupReduceITCase {
- @Parameters
- def getConfigurations: java.util.Collection[Array[AnyRef]] = {
- val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to GroupReduceProgs.NUM_PROGRAMS) {
- val config = new Configuration()
- config.setInteger("ProgramId", i)
- configs += Array(config)
- }
-
- configs.asJavaCollection
- }
-}
-
-class NestedTupleReducer extends GroupReduceFunction[((Int, Int), String), String] {
- def reduce(values: Iterable[((Int, Int), String)], out: Collector[String]) {
- var once: Boolean = false
- val concat: StringBuilder = new StringBuilder
- for (value <- values.asScala) {
- if (!once) {
- concat.append(value._2).append("--")
- once = true
- }
- concat.append(value._1)
- concat.append("-")
- }
- out.collect(concat.toString())
- }
-}
-
-