You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2014/10/08 11:40:21 UTC
[01/11] Really add POJO support and nested keys for Scala API
Repository: incubator-flink
Updated Branches:
refs/heads/master ec82d973d -> 6be855543
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
index dd1ac99..fe1dd43 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupingTest.scala
@@ -17,10 +17,10 @@
*/
package org.apache.flink.api.scala.operators
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
import org.junit.Assert
import org.apache.flink.api.common.InvalidProgramException
import org.apache.flink.api.common.operators.Order
-import org.junit.Ignore
import org.junit.Test
import org.apache.flink.api.scala._
@@ -96,7 +96,7 @@ class GroupingTest {
}
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[IllegalArgumentException])
def testGroupByKeyFields2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val longDs = env.fromCollection(emptyLongData)
@@ -105,7 +105,7 @@ class GroupingTest {
longDs.groupBy("_1")
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[IllegalArgumentException])
def testGroupByKeyFields3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val customDs = env.fromCollection(customTypeData)
@@ -114,7 +114,7 @@ class GroupingTest {
customDs.groupBy("_1")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testGroupByKeyFields4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tupleDs = env.fromCollection(emptyTupleData)
@@ -123,7 +123,15 @@ class GroupingTest {
tupleDs.groupBy("foo")
}
- @Ignore
+ @Test
+ def testGroupByKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should not work
+ customDs.groupBy("myInt")
+ }
+
@Test
def testGroupByKeyExpressions1(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -131,24 +139,22 @@ class GroupingTest {
// should work
try {
-// ds.groupBy("i");
+ ds.groupBy("myInt")
}
catch {
case e: Exception => Assert.fail()
}
}
- @Ignore
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[IllegalArgumentException])
def testGroupByKeyExpressions2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
// should not work: groups on basic type
-// longDs.groupBy("l");
val longDs = env.fromCollection(emptyLongData)
+ longDs.groupBy("l")
}
- @Ignore
@Test(expected = classOf[InvalidProgramException])
def testGroupByKeyExpressions3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -158,14 +164,13 @@ class GroupingTest {
customDs.groupBy(0)
}
- @Ignore
@Test(expected = classOf[IllegalArgumentException])
def testGroupByKeyExpressions4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds = env.fromCollection(customTypeData)
// should not work, non-existent field
-// ds.groupBy("myNonExistent");
+ ds.groupBy("myNonExistent")
}
@Test
@@ -173,7 +178,7 @@ class GroupingTest {
val env = ExecutionEnvironment.getExecutionEnvironment
try {
val customDs = env.fromCollection(customTypeData)
- customDs.groupBy { _.l }
+ customDs.groupBy { _.myLong }
}
catch {
case e: Exception => Assert.fail()
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
index cae936d..0219154 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
@@ -18,6 +18,7 @@
package org.apache.flink.api.scala.operators
import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
import org.junit.Assert
import org.apache.flink.api.common.InvalidProgramException
import org.junit.Ignore
@@ -132,7 +133,7 @@ class JoinOperatorTest {
ds1.join(ds2).where("_1", "_2").equalTo("_3")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testJoinKeyFields4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -142,7 +143,7 @@ class JoinOperatorTest {
ds1.join(ds2).where("foo").equalTo("_1")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testJoinKeyFields5(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -152,7 +153,7 @@ class JoinOperatorTest {
ds1.join(ds2).where("_1").equalTo("bar")
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[IllegalArgumentException])
def testJoinKeyFields6(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -162,7 +163,6 @@ class JoinOperatorTest {
ds1.join(ds2).where("_2").equalTo("_1")
}
- @Ignore
@Test
def testJoinKeyExpressions1(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -171,36 +171,33 @@ class JoinOperatorTest {
// should work
try {
-// ds1.join(ds2).where("i").equalTo("i")
+ ds1.join(ds2).where("myInt").equalTo("myInt")
}
catch {
case e: Exception => Assert.fail()
}
}
- @Ignore
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyExpressions2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(customTypeData)
val ds2 = env.fromCollection(customTypeData)
// should not work, incompatible join key types
-// ds1.join(ds2).where("i").equalTo("s")
+ ds1.join(ds2).where("myInt").equalTo("myString")
}
- @Ignore
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyExpressions3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(customTypeData)
val ds2 = env.fromCollection(customTypeData)
// should not work, incompatible number of keys
-// ds1.join(ds2).where("i", "s").equalTo("i")
+ ds1.join(ds2).where("myInt", "myString").equalTo("myInt")
}
- @Ignore
@Test(expected = classOf[IllegalArgumentException])
def testJoinKeyExpressions4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -208,7 +205,7 @@ class JoinOperatorTest {
val ds2 = env.fromCollection(customTypeData)
// should not work, join key non-existent
-// ds1.join(ds2).where("myNonExistent").equalTo("i")
+ ds1.join(ds2).where("myNonExistent").equalTo("i")
}
@Test
@@ -219,7 +216,7 @@ class JoinOperatorTest {
// should work
try {
- ds1.join(ds2).where { _.l} equalTo { _.l }
+ ds1.join(ds2).where { _.myLong} equalTo { _.myLong }
}
catch {
case e: Exception => Assert.fail()
@@ -234,7 +231,7 @@ class JoinOperatorTest {
// should work
try {
- ds1.join(ds2).where { _.l }.equalTo(3)
+ ds1.join(ds2).where { _.myLong }.equalTo(3)
}
catch {
case e: Exception => Assert.fail()
@@ -249,7 +246,7 @@ class JoinOperatorTest {
// should work
try {
- ds1.join(ds2).where(3).equalTo { _.l }
+ ds1.join(ds2).where(3).equalTo { _.myLong }
}
catch {
case e: Exception => Assert.fail()
@@ -263,7 +260,7 @@ class JoinOperatorTest {
val ds2 = env.fromCollection(customTypeData)
// should not work, incompatible types
- ds1.join(ds2).where(2).equalTo { _.l }
+ ds1.join(ds2).where(2).equalTo { _.myLong }
}
@Test(expected = classOf[IncompatibleKeysException])
@@ -273,7 +270,7 @@ class JoinOperatorTest {
val ds2 = env.fromCollection(customTypeData)
// should not work, more than one field position key
- ds1.join(ds2).where(1, 3) equalTo { _.l }
+ ds1.join(ds2).where(1, 3) equalTo { _.myLong }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
index a8447a9..bea91df 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/PartitionITCase.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.operators
import org.apache.flink.api.common.functions.{RichFilterFunction, RichMapFunction}
import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.configuration.Configuration
import org.apache.flink.test.util.JavaProgramTestBase
import org.junit.runner.RunWith
@@ -32,40 +33,18 @@ import org.apache.flink.api.scala._
object PartitionProgs {
- var NUM_PROGRAMS: Int = 6
-
- val tupleInput = Array(
- (1, "Foo"),
- (1, "Foo"),
- (1, "Foo"),
- (2, "Foo"),
- (2, "Foo"),
- (2, "Foo"),
- (2, "Foo"),
- (2, "Foo"),
- (3, "Foo"),
- (3, "Foo"),
- (3, "Foo"),
- (4, "Foo"),
- (4, "Foo"),
- (4, "Foo"),
- (4, "Foo"),
- (5, "Foo"),
- (5, "Foo"),
- (6, "Foo"),
- (6, "Foo"),
- (6, "Foo"),
- (6, "Foo")
- )
-
+ var NUM_PROGRAMS: Int = 7
def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
progId match {
case 1 =>
+ /*
+ * Test hash partition by tuple field
+ */
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
- val unique = ds.partitionByHash(0).mapPartition( _.map(_._1).toSet )
+ val unique = ds.partitionByHash(1).mapPartition( _.map(_._2).toSet )
unique.writeAsText(resultPath)
env.execute()
@@ -73,16 +52,22 @@ object PartitionProgs {
"1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
case 2 =>
+ /*
+ * Test hash partition by key selector
+ */
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
- val unique = ds.partitionByHash( _._1 ).mapPartition( _.map(_._1).toSet )
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val unique = ds.partitionByHash( _._2 ).mapPartition( _.map(_._2).toSet )
unique.writeAsText(resultPath)
env.execute()
"1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
case 3 =>
- val env = ExecutionEnvironment.getExecutionEnvironment
+ /*
+ * Test forced rebalancing
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
val ds = env.generateSequence(1, 3000)
val skewed = ds.filter(_ > 780)
@@ -101,8 +86,8 @@ object PartitionProgs {
countsInPartition.writeAsText(resultPath)
env.execute()
- val numPerPartition : Int = 2220 / env.getDegreeOfParallelism / 10;
- var result = "";
+ val numPerPartition : Int = 2220 / env.getDegreeOfParallelism / 10
+ var result = ""
for (i <- 0 until env.getDegreeOfParallelism) {
result += "(" + i + "," + numPerPartition + ")\n"
}
@@ -112,10 +97,12 @@ object PartitionProgs {
// Verify that mapPartition operation after repartition picks up correct
// DOP
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
env.setDegreeOfParallelism(1)
- val unique = ds.partitionByHash(0).setParallelism(4).mapPartition( _.map(_._1).toSet )
+ val unique = ds.partitionByHash(1)
+ .setParallelism(4)
+ .mapPartition( _.map(_._2).toSet )
unique.writeAsText(resultPath)
env.execute()
@@ -126,13 +113,13 @@ object PartitionProgs {
// Verify that map operation after repartition picks up correct
// DOP
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
env.setDegreeOfParallelism(1)
val count = ds.partitionByHash(0).setParallelism(4).map(
- new RichMapFunction[(Int, String), Tuple1[Int]] {
+ new RichMapFunction[(Int, Long, String), Tuple1[Int]] {
var first = true
- override def map(in: (Int, String)): Tuple1[Int] = {
+ override def map(in: (Int, Long, String)): Tuple1[Int] = {
// only output one value with count 1
if (first) {
first = false
@@ -152,13 +139,13 @@ object PartitionProgs {
// Verify that filter operation after repartition picks up correct
// DOP
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
env.setDegreeOfParallelism(1)
val count = ds.partitionByHash(0).setParallelism(4).filter(
- new RichFilterFunction[(Int, String)] {
+ new RichFilterFunction[(Int, Long, String)] {
var first = true
- override def filter(in: (Int, String)): Boolean = {
+ override def filter(in: (Int, Long, String)): Boolean = {
// only output one value with count 1
if (first) {
first = false
@@ -175,6 +162,19 @@ object PartitionProgs {
if (onCollection) "(1)\n" else "(4)\n"
+ case 7 =>
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(3)
+ val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
+ val uniqLongs = ds
+ .partitionByHash("nestedPojo.longNumber")
+ .setParallelism(4)
+ .mapPartition( _.map(_.nestedPojo.longNumber).toSet )
+
+ uniqLongs.writeAsText(resultPath)
+ env.execute()
+ "10000\n" + "20000\n" + "30000\n"
+
case _ =>
throw new IllegalArgumentException("Invalid program id")
}
@@ -194,7 +194,7 @@ class PartitionITCase(config: Configuration) extends JavaProgramTestBase(config)
}
protected def testProgram(): Unit = {
- expectedResult = PartitionProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ expectedResult = GroupReduceProgs.runProgram(curProgId, resultPath, isCollectionExecution)
}
protected override def postSubmit(): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
index d33da41..2b2d3a9 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/types/TypeInformationGenTest.scala
@@ -44,7 +44,9 @@ class CustomType(var myField1: String, var myField2: Int) {
}
}
-class MyObject[A](var a: A)
+class MyObject[A](var a: A) {
+ def this() { this(null.asInstanceOf[A]) }
+}
class TypeInformationGenTest {
@@ -139,7 +141,7 @@ class TypeInformationGenTest {
Assert.assertFalse(ti.isBasicType)
Assert.assertFalse(ti.isTupleType)
- Assert.assertTrue(ti.isInstanceOf[GenericTypeInfo[_]])
+ Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
Assert.assertEquals(ti.getTypeClass, classOf[CustomType])
}
@@ -152,7 +154,7 @@ class TypeInformationGenTest {
val tti = ti.asInstanceOf[TupleTypeInfoBase[_]]
Assert.assertEquals(classOf[Tuple2[_, _]], tti.getTypeClass)
Assert.assertEquals(classOf[java.lang.Long], tti.getTypeAt(0).getTypeClass)
- Assert.assertTrue(tti.getTypeAt(1).isInstanceOf[GenericTypeInfo[_]])
+ Assert.assertTrue(tti.getTypeAt(1).isInstanceOf[PojoTypeInfo[_]])
Assert.assertEquals(classOf[CustomType], tti.getTypeAt(1).getTypeClass)
}
@@ -235,7 +237,7 @@ class TypeInformationGenTest {
def testParamertizedCustomObject(): Unit = {
val ti = createTypeInformation[MyObject[String]]
- Assert.assertTrue(ti.isInstanceOf[GenericTypeInfo[_]])
+ Assert.assertTrue(ti.isInstanceOf[PojoTypeInfo[_]])
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
new file mode 100644
index 0000000..60f86a0
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/util/CollectionDataSets.scala
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.util
+
+import org.apache.hadoop.io.IntWritable
+
+import org.apache.flink.api.scala._
+
+import scala.collection.mutable
+import scala.util.Random
+
+/**
+ * #################################################################################################
+ *
+ * BE AWARE THAT OTHER TESTS DEPEND ON THIS TEST DATA.
+ * IF YOU MODIFY THE DATA MAKE SURE YOU CHECK THAT ALL TESTS ARE STILL WORKING!
+ *
+ * #################################################################################################
+ */
+object CollectionDataSets {
+ def get3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ data.+=((4, 3L, "Hello world, how are you?"))
+ data.+=((5, 3L, "I am fine."))
+ data.+=((6, 3L, "Luke Skywalker"))
+ data.+=((7, 4L, "Comment#1"))
+ data.+=((8, 4L, "Comment#2"))
+ data.+=((9, 4L, "Comment#3"))
+ data.+=((10, 4L, "Comment#4"))
+ data.+=((11, 5L, "Comment#5"))
+ data.+=((12, 5L, "Comment#6"))
+ data.+=((13, 5L, "Comment#7"))
+ data.+=((14, 5L, "Comment#8"))
+ data.+=((15, 5L, "Comment#9"))
+ data.+=((16, 6L, "Comment#10"))
+ data.+=((17, 6L, "Comment#11"))
+ data.+=((18, 6L, "Comment#12"))
+ data.+=((19, 6L, "Comment#13"))
+ data.+=((20, 6L, "Comment#14"))
+ data.+=((21, 6L, "Comment#15"))
+ Random.shuffle(data)
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmall3TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def get5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
+ val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
+ data.+=((1, 1L, 0, "Hallo", 1L))
+ data.+=((2, 2L, 1, "Hallo Welt", 2L))
+ data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
+ data.+=((3, 4L, 3, "Hallo Welt wie gehts?", 2L))
+ data.+=((3, 5L, 4, "ABC", 2L))
+ data.+=((3, 6L, 5, "BCD", 3L))
+ data.+=((4, 7L, 6, "CDE", 2L))
+ data.+=((4, 8L, 7, "DEF", 1L))
+ data.+=((4, 9L, 8, "EFG", 1L))
+ data.+=((4, 10L, 9, "FGH", 2L))
+ data.+=((5, 11L, 10, "GHI", 1L))
+ data.+=((5, 12L, 11, "HIJ", 3L))
+ data.+=((5, 13L, 12, "IJK", 3L))
+ data.+=((5, 14L, 13, "JKL", 2L))
+ data.+=((5, 15L, 14, "KLM", 2L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmall5TupleDataSet(env: ExecutionEnvironment): DataSet[(Int, Long, Int, String, Long)] = {
+ val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
+ data.+=((1, 1L, 0, "Hallo", 1L))
+ data.+=((2, 2L, 1, "Hallo Welt", 2L))
+ data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
+ val data = new mutable.MutableList[((Int, Int), String)]
+ data.+=(((1, 1), "one"))
+ data.+=(((2, 2), "two"))
+ data.+=(((3, 3), "three"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getGroupSortedNestedTupleDataSet(env: ExecutionEnvironment): DataSet[((Int, Int), String)] = {
+ val data = new mutable.MutableList[((Int, Int), String)]
+ data.+=(((1, 3), "a"))
+ data.+=(((1, 2), "a"))
+ data.+=(((2, 1), "a"))
+ data.+=(((2, 2), "b"))
+ data.+=(((3, 3), "c"))
+ data.+=(((3, 6), "c"))
+ data.+=(((4, 9), "c"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getStringDataSet(env: ExecutionEnvironment): DataSet[String] = {
+ val data = new mutable.MutableList[String]
+ data.+=("Hi")
+ data.+=("Hello")
+ data.+=("Hello world")
+ data.+=("Hello world, how are you?")
+ data.+=("I am fine.")
+ data.+=("Luke Skywalker")
+ data.+=("Random comment")
+ data.+=("LOL")
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getIntDataSet(env: ExecutionEnvironment): DataSet[Int] = {
+ val data = new mutable.MutableList[Int]
+ data.+=(1)
+ data.+=(2)
+ data.+=(2)
+ data.+=(3)
+ data.+=(3)
+ data.+=(3)
+ data.+=(4)
+ data.+=(4)
+ data.+=(4)
+ data.+=(4)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ data.+=(5)
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
+ val data = new mutable.MutableList[CustomType]
+ data.+=(new CustomType(1, 0L, "Hi"))
+ data.+=(new CustomType(2, 1L, "Hello"))
+ data.+=(new CustomType(2, 2L, "Hello world"))
+ data.+=(new CustomType(3, 3L, "Hello world, how are you?"))
+ data.+=(new CustomType(3, 4L, "I am fine."))
+ data.+=(new CustomType(3, 5L, "Luke Skywalker"))
+ data.+=(new CustomType(4, 6L, "Comment#1"))
+ data.+=(new CustomType(4, 7L, "Comment#2"))
+ data.+=(new CustomType(4, 8L, "Comment#3"))
+ data.+=(new CustomType(4, 9L, "Comment#4"))
+ data.+=(new CustomType(5, 10L, "Comment#5"))
+ data.+=(new CustomType(5, 11L, "Comment#6"))
+ data.+=(new CustomType(5, 12L, "Comment#7"))
+ data.+=(new CustomType(5, 13L, "Comment#8"))
+ data.+=(new CustomType(5, 14L, "Comment#9"))
+ data.+=(new CustomType(6, 15L, "Comment#10"))
+ data.+=(new CustomType(6, 16L, "Comment#11"))
+ data.+=(new CustomType(6, 17L, "Comment#12"))
+ data.+=(new CustomType(6, 18L, "Comment#13"))
+ data.+=(new CustomType(6, 19L, "Comment#14"))
+ data.+=(new CustomType(6, 20L, "Comment#15"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallCustomTypeDataSet(env: ExecutionEnvironment): DataSet[CustomType] = {
+ val data = new mutable.MutableList[CustomType]
+ data.+=(new CustomType(1, 0L, "Hi"))
+ data.+=(new CustomType(2, 1L, "Hello"))
+ data.+=(new CustomType(2, 2L, "Hello world"))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallTuplebasedPojoMatchingDataSet(env: ExecutionEnvironment):
+ DataSet[(Int, String, Int, Int, Long, String, Long)] = {
+ val data = new mutable.MutableList[(Int, String, Int, Int, Long, String, Long)]
+ data.+=((1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=((2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=((3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getSmallPojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
+ val data = new mutable.MutableList[POJO]
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(Random.shuffle(data))
+ }
+
+ def getDuplicatePojoDataSet(env: ExecutionEnvironment): DataSet[POJO] = {
+ val data = new mutable.MutableList[POJO]
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(1, "First", 10, 100, 1000L, "One", 10000L))
+ data.+=(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ data.+=(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L))
+ env.fromCollection(data)
+ }
+
+ def getCrazyNestedDataSet(env: ExecutionEnvironment): DataSet[CrazyNested] = {
+ val data = new mutable.MutableList[CrazyNested]
+ data.+=(new CrazyNested("aa"))
+ data.+=(new CrazyNested("bb"))
+ data.+=(new CrazyNested("bb"))
+ data.+=(new CrazyNested("cc"))
+ data.+=(new CrazyNested("cc"))
+ data.+=(new CrazyNested("cc"))
+ env.fromCollection(data)
+ }
+
+ def getPojoContainingTupleAndWritable(env: ExecutionEnvironment): DataSet[CollectionDataSets
+ .PojoContainingTupleAndWritable] = {
+ val data = new
+ mutable.MutableList[PojoContainingTupleAndWritable]
+ data.+=(new PojoContainingTupleAndWritable(1, 10L, 100L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ data.+=(new PojoContainingTupleAndWritable(2, 20L, 200L))
+ env.fromCollection(data)
+ }
+
+ def getTupleContainingPojos(env: ExecutionEnvironment): DataSet[(Int, CrazyNested, POJO)] = {
+ val data = new mutable.MutableList[(Int, CrazyNested, POJO)]
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 1,
+ new CrazyNested("one", "uno", 1L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ data.+=((
+ 2,
+ new CrazyNested("two", "duo", 2L),
+ new POJO(1, "First", 10, 100, 1000L, "One", 10000L)))
+ env.fromCollection(data)
+ }
+
+ def getPojoWithMultiplePojos(env: ExecutionEnvironment): DataSet[CollectionDataSets
+ .PojoWithMultiplePojos] = {
+ val data = new mutable.MutableList[CollectionDataSets
+ .PojoWithMultiplePojos]
+ data.+=(new PojoWithMultiplePojos("a", "aa", "b", "bb", 1))
+ data.+=(new PojoWithMultiplePojos("b", "bb", "c", "cc", 2))
+ data.+=(new PojoWithMultiplePojos("d", "dd", "e", "ee", 3))
+ env.fromCollection(data)
+ }
+
+
+ class CustomType(var myInt: Int, var myLong: Long, var myString: String) {
+ def this() {
+ this(0, 0, "")
+ }
+
+ override def toString: String = {
+ myInt + "," + myLong + "," + myString
+ }
+ }
+
+ class POJO(
+ var number: Int,
+ var str: String,
+ var nestedTupleWithCustom: (Int, CustomType),
+ var nestedPojo: NestedPojo) {
+ def this() {
+ this(0, "", null, null)
+ }
+
+ def this(i0: Int, s0: String, i1: Int, i2: Int, l0: Long, s1: String, l1: Long) {
+ this(i0, s0, (i1, new CustomType(i2, l0, s1)), new NestedPojo(l1))
+ }
+
+ override def toString: String = {
+ number + " " + str + " " + nestedTupleWithCustom + " " + nestedPojo.longNumber
+ }
+
+ @transient var ignoreMe: Long = 1L
+ }
+
+ class NestedPojo(var longNumber: Long) {
+ def this() {
+ this(0)
+ }
+ }
+
+ class CrazyNested(var nest_Lvl1: CrazyNestedL1, var something: Long) {
+ def this() {
+ this(new CrazyNestedL1, 0)
+ }
+
+ def this(set: String) {
+ this()
+ nest_Lvl1 = new CrazyNestedL1
+ nest_Lvl1.nest_Lvl2 = new CrazyNestedL2
+ nest_Lvl1.nest_Lvl2.nest_Lvl3 = new CrazyNestedL3
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4 = new CrazyNestedL4
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal = set
+ }
+
+ def this(set: String, second: String, s: Long) {
+ this(set)
+ something = s
+ nest_Lvl1.a = second
+ }
+ }
+
+ class CrazyNestedL1 {
+ var a: String = null
+ var b: Int = 0
+ var nest_Lvl2: CrazyNestedL2 = null
+ }
+
+ class CrazyNestedL2 {
+ var nest_Lvl3: CrazyNestedL3 = null
+ }
+
+ class CrazyNestedL3 {
+ var nest_Lvl4: CrazyNestedL4 = null
+ }
+
+ class CrazyNestedL4 {
+ var f1nal: String = null
+ }
+
+ class PojoContainingTupleAndWritable(
+ var someInt: Int,
+ var someString: String,
+ var hadoopFan: IntWritable,
+ var theTuple: (Long, Long)) {
+ def this() {
+ this(0, "", new IntWritable(0), (0, 0))
+ }
+
+ def this(i: Int, l1: Long, l2: Long) {
+ this()
+ hadoopFan = new IntWritable(i)
+ someInt = i
+ theTuple = (l1, l2)
+ }
+
+ }
+
+ class Pojo1 {
+ var a: String = null
+ var b: String = null
+ }
+
+ class Pojo2 {
+ var a2: String = null
+ var b2: String = null
+ }
+
+ class PojoWithMultiplePojos {
+
+ def this(a: String, b: String, a1: String, b1: String, i0: Int) {
+ this()
+ p1 = new Pojo1
+ p1.a = a
+ p1.b = b
+ p2 = new Pojo2
+ p2.a2 = a1
+ p2.a2 = b1
+ this.i0 = i0
+ }
+
+ var p1: Pojo1 = null
+ var p2: Pojo2 = null
+ var i0: Int = 0
+ }
+
+}
+
[04/11] [FLINK-1032] Rework support for POJO types in the Java API
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
deleted file mode 100644
index 0a5732c..0000000
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountPOJOITCase.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-//
-//package org.apache.flink.test.exampleJavaPrograms;
-//
-////import org.apache.flink.examples.java.wordcount.WordCountPOJO;
-//import org.apache.flink.test.testdata.WordCountData;
-//import org.apache.flink.test.util.JavaProgramTestBase;
-//import org.junit.Ignore;
-//
-//@Ignore
-//public class WordCountPOJOITCase extends JavaProgramTestBase {
-//
-// protected String textPath;
-// protected String resultPath;
-//
-//
-// @Override
-// protected void preSubmit() throws Exception {
-// textPath = createTempFile("text.txt", WordCountData.TEXT);
-// resultPath = getTempDirPath("result");
-// }
-//
-// @Override
-// protected void postSubmit() throws Exception {
-// compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath);
-// }
-//
-// @Override
-// protected void testProgram() throws Exception {
-// WordCountPOJO.main(new String[]{textPath, resultPath});
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountSimplePOJOITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountSimplePOJOITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountSimplePOJOITCase.java
new file mode 100644
index 0000000..7d20597
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountSimplePOJOITCase.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.exampleJavaPrograms;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.util.JavaProgramTestBase;
+import org.apache.flink.util.Collector;
+
+import java.io.Serializable;
+
+
+public class WordCountSimplePOJOITCase extends JavaProgramTestBase implements Serializable {
+ private static final long serialVersionUID = 1L;
+ protected String textPath;
+ protected String resultPath;
+
+
+ @Override
+ protected void preSubmit() throws Exception {
+ textPath = createTempFile("text.txt", WordCountData.TEXT);
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath);
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<String> text = env.readTextFile(textPath);
+
+ DataSet<WC> counts = text
+ .flatMap(new Tokenizer())
+ .groupBy("word")
+ .reduce(new ReduceFunction<WC>() {
+ private static final long serialVersionUID = 1L;
+
+ public WC reduce(WC value1, WC value2) {
+ return new WC(value1.word, value1.count + value2.count);
+ }
+ });
+
+ counts.writeAsText(resultPath);
+
+ env.execute("WordCount with custom data types example");
+ }
+
+ public static final class Tokenizer implements FlatMapFunction<String, WC> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<WC> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new WC(token, 1));
+ }
+ }
+ }
+ }
+
+ public static class WC {
+ public WC() {}
+ public WC(String w, int c) {
+ word = w;
+ count = c;
+ }
+ public String word;
+ public int count;
+ @Override
+ public String toString() {
+ return word + " " + count;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
index 9701086..4f00b1d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
@@ -18,7 +18,7 @@
package org.apache.flink.test.exampleScalaPrograms;
-import org.apache.flink.examples.scala.graph.EnumTrianglesOpt;
+import org.apache.flink.examples.java.graph.EnumTrianglesOpt;
import org.apache.flink.test.testdata.EnumTriangleData;
import org.apache.flink.test.util.JavaProgramTestBase;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
index d14c2e1..42c6a8e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
@@ -18,19 +18,19 @@
package org.apache.flink.test.exampleScalaPrograms;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.examples.scala.graph.PageRankBasic;
+import org.apache.flink.examples.java.graph.PageRankBasic;
import org.apache.flink.test.testdata.PageRankData;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.LinkedList;
-
@RunWith(Parameterized.class)
public class PageRankITCase extends JavaProgramTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithAdjacencyListComparator.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithAdjacencyListComparator.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithAdjacencyListComparator.java
index ce31ea2..e607ba7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithAdjacencyListComparator.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithAdjacencyListComparator.java
@@ -32,8 +32,6 @@ public final class VertexWithAdjacencyListComparator extends TypeComparator<Vert
private long reference;
- private Comparable[] extractedKey = new Comparable[1];
-
private TypeComparator[] comparators = new TypeComparator[]{new LongComparator(true)};
@Override
@@ -136,13 +134,13 @@ public final class VertexWithAdjacencyListComparator extends TypeComparator<Vert
}
@Override
- public Object[] extractKeys(VertexWithAdjacencyList record) {
- extractedKey[0] = record.getVertexID();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = ((VertexWithAdjacencyList) record).getVertexID();
+ return 1;
}
@Override
- public TypeComparator[] getComparators() {
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankAndDanglingComparator.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankAndDanglingComparator.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankAndDanglingComparator.java
index b9503ae..dd9d4dd 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankAndDanglingComparator.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankAndDanglingComparator.java
@@ -32,8 +32,6 @@ public final class VertexWithRankAndDanglingComparator extends TypeComparator<Ve
private long reference;
- private Comparable[] extractedKey = new Comparable[1];
-
private TypeComparator[] comparators = new TypeComparator[]{new LongComparator(true)};
@Override
@@ -141,13 +139,13 @@ public final class VertexWithRankAndDanglingComparator extends TypeComparator<Ve
}
@Override
- public Object[] extractKeys(VertexWithRankAndDangling record) {
- extractedKey[0] = record.getVertexID();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = ((VertexWithRankAndDangling) record).getVertexID();
+ return 1;
}
@Override
- public TypeComparator[] getComparators() {
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankComparator.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankComparator.java b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankComparator.java
index cfe4a9f..b63f724 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankComparator.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/iterative/nephele/customdanglingpagerank/types/VertexWithRankComparator.java
@@ -32,8 +32,6 @@ public final class VertexWithRankComparator extends TypeComparator<VertexWithRan
private long reference;
- private Comparable[] extractedKey = new Comparable[1];
-
private TypeComparator[] comparators = new TypeComparator[]{new LongComparator(true)};
@Override
@@ -139,13 +137,13 @@ public final class VertexWithRankComparator extends TypeComparator<VertexWithRan
}
@Override
- public Object[] extractKeys(VertexWithRank record) {
- extractedKey[0] = record.getVertexID();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = ((VertexWithRank) record).getVertexID();
+ return 1;
}
@Override
- public TypeComparator[] getComparators() {
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index ce9aa65..f0e89df 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -26,18 +26,22 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.common.functions.RichCoGroupFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple7;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
+import org.junit.Assert;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -45,7 +49,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class CoGroupITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 9;
+ private static int NUM_PROGRAMS = 13;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -356,6 +360,149 @@ public class CoGroupITCase extends JavaProgramTestBase {
"5,3,HIJ\n" +
"5,3,IJK\n";
}
+ case 10: {
+ /*
+ * CoGroup on two custom type inputs using expression keys
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
+ DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
+ DataSet<CustomType> coGroupDs = ds.coGroup(ds2).where("myInt").equalTo("myInt").with(new CustomTypeCoGroup());
+
+ coGroupDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,0,test\n" +
+ "2,6,test\n" +
+ "3,24,test\n" +
+ "4,60,test\n" +
+ "5,120,test\n" +
+ "6,210,test\n";
+ }
+ case 11: {
+ /*
+ * CoGroup on two custom type inputs using expression keys
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
+ .where("nestedPojo.longNumber").equalTo(6).with(new CoGroupFunction<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long>, CustomType>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coGroup(
+ Iterable<POJO> first,
+ Iterable<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> second,
+ Collector<CustomType> out) throws Exception {
+ for(POJO p : first) {
+ for(Tuple7<Integer, String, Integer, Integer, Long, String, Long> t: second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t.f6);
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"));
+ }
+ }
+ }
+ });
+ coGroupDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "-1,20000,Flink\n" +
+ "-1,10000,Flink\n" +
+ "-1,30000,Flink\n";
+ }
+ case 12: {
+ /*
+ * CoGroup field-selector (expression keys) + key selector function
+ * The key selector is unnecessary complicated (Tuple1) ;)
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
+ .where(new KeySelector<POJO, Tuple1<Long>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple1<Long> getKey(POJO value)
+ throws Exception {
+ return new Tuple1<Long>(value.nestedPojo.longNumber);
+ }
+ }).equalTo(6).with(new CoGroupFunction<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long>, CustomType>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coGroup(
+ Iterable<POJO> first,
+ Iterable<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> second,
+ Collector<CustomType> out) throws Exception {
+ for(POJO p : first) {
+ for(Tuple7<Integer, String, Integer, Integer, Long, String, Long> t: second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t.f6);
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"));
+ }
+ }
+ }
+ });
+ coGroupDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "-1,20000,Flink\n" +
+ "-1,10000,Flink\n" +
+ "-1,30000,Flink\n";
+ }
+ case 13: {
+ /*
+ * CoGroup field-selector (expression keys) + key selector function
+ * The key selector is simple here
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
+ .where(new KeySelector<POJO, Long>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Long getKey(POJO value)
+ throws Exception {
+ return value.nestedPojo.longNumber;
+ }
+ }).equalTo(6).with(new CoGroupFunction<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long>, CustomType>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void coGroup(
+ Iterable<POJO> first,
+ Iterable<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> second,
+ Collector<CustomType> out) throws Exception {
+ for(POJO p : first) {
+ for(Tuple7<Integer, String, Integer, Integer, Long, String, Long> t: second) {
+ Assert.assertTrue(p.nestedPojo.longNumber == t.f6);
+ out.collect(new CustomType(-1, p.nestedPojo.longNumber, "Flink"));
+ }
+ }
+ }
+ });
+ coGroupDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "-1,20000,Flink\n" +
+ "-1,10000,Flink\n" +
+ "-1,30000,Flink\n";
+ }
+
default:
throw new IllegalArgumentException("Invalid program id");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index 00b68fc..fce56d1 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -35,7 +35,12 @@ import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.compiler.PactCompiler;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CrazyNested;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.FromTuple;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.FromTupleWithCTor;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.PojoContainingTupleAndWritable;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
import org.junit.runner.RunWith;
@@ -48,7 +53,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
@RunWith(Parameterized.class)
public class GroupReduceITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 15;
+ private static int NUM_PROGRAMS = 19;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -406,7 +411,6 @@ public class GroupReduceITCase extends JavaProgramTestBase {
/*
* check correctness of groupReduce with descending group sort
*/
-
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setDegreeOfParallelism(1);
@@ -484,6 +488,117 @@ public class GroupReduceITCase extends JavaProgramTestBase {
"16,6,Comment#10\n";
}
+ case 16: {
+ /*
+ * Deep nesting test
+ * + null value in pojo
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<CrazyNested> ds = CollectionDataSets.getCrazyNestedDataSet(env);
+ DataSet<Tuple2<String, Integer>> reduceDs = ds.groupBy("nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal")
+ .reduceGroup(new GroupReduceFunction<CollectionDataSets.CrazyNested, Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void reduce(Iterable<CrazyNested> values,
+ Collector<Tuple2<String, Integer>> out)
+ throws Exception {
+ int c = 0; String n = null;
+ for(CrazyNested v : values) {
+ c++; // haha
+ n = v.nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal;
+ }
+ out.collect(new Tuple2<String, Integer>(n,c));
+ }});
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "aa,1\nbb,2\ncc,3\n";
+ }
+ case 17: {
+ /*
+ * Test Pojo extending from tuple WITH custom fields
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<FromTupleWithCTor> ds = CollectionDataSets.getPojoExtendingFromTuple(env);
+ DataSet<Integer> reduceDs = ds.groupBy("special", "f2")
+ .reduceGroup(new GroupReduceFunction<FromTupleWithCTor, Integer>() {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public void reduce(Iterable<FromTupleWithCTor> values,
+ Collector<Integer> out)
+ throws Exception {
+ int c = 0;
+ for(FromTuple v : values) {
+ c++;
+ }
+ out.collect(c);
+ }});
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "3\n2\n";
+ }
+ case 18: {
+ /*
+ * Test Pojo containing a Writable and Tuples
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getPojoContainingTupleAndWritable(env);
+ DataSet<Integer> reduceDs = ds.groupBy("hadoopFan", "theTuple.*") // full tuple selection
+ .reduceGroup(new GroupReduceFunction<PojoContainingTupleAndWritable, Integer>() {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public void reduce(Iterable<PojoContainingTupleAndWritable> values,
+ Collector<Integer> out)
+ throws Exception {
+ int c = 0;
+ for(PojoContainingTupleAndWritable v : values) {
+ c++;
+ }
+ out.collect(c);
+ }});
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1\n5\n";
+ }
+ case 19: {
+ /*
+ * Test Tuple containing pojos and regular fields
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer,CrazyNested, POJO>> ds = CollectionDataSets.getTupleContainingPojos(env);
+ DataSet<Integer> reduceDs = ds.groupBy("f0", "f1.*") // nested full tuple selection
+ .reduceGroup(new GroupReduceFunction<Tuple3<Integer,CrazyNested, POJO>, Integer>() {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public void reduce(Iterable<Tuple3<Integer,CrazyNested, POJO>> values,
+ Collector<Integer> out)
+ throws Exception {
+ int c = 0;
+ for(Tuple3<Integer,CrazyNested, POJO> v : values) {
+ c++;
+ }
+ out.collect(c);
+ }});
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "3\n1\n";
+ }
default: {
throw new IllegalArgumentException("Invalid program id");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
index 433a076..e8d8be9 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
@@ -31,9 +31,11 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
import org.junit.runner.RunWith;
@@ -46,7 +48,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
@RunWith(Parameterized.class)
public class JoinITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 14;
+ private static int NUM_PROGRAMS = 21;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -493,8 +495,175 @@ public class JoinITCase extends JavaProgramTestBase {
"I am fine.,HIJ\n" +
"I am fine.,IJK\n";
}
+ /**
+ * Joins with POJOs
+ */
+ case 15: {
+ /*
+ * Join nested pojo against tuple (selected using a string)
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("nestedPojo.longNumber").equalTo("f6");
+
+ joinDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+ }
+
+ case 16: {
+ /*
+ * Join nested pojo against tuple (selected as an integer)
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("nestedPojo.longNumber").equalTo(6); // <--- difference!
+
+ joinDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+ }
+ case 17: {
+ /*
+ * selecting multiple fields using expression language
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("nestedPojo.longNumber", "number", "str").equalTo("f6","f0","f1");
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+
+ }
+ case 18: {
+ /*
+ * nested into tuple
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("nestedPojo.longNumber", "number","nestedTupleWithCustom.f0").equalTo("f6","f0","f2");
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+
+ }
+ case 19: {
+ /*
+ * nested into tuple into pojo
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("nestedTupleWithCustom.f0","nestedTupleWithCustom.f1.myInt","nestedTupleWithCustom.f1.myLong").equalTo("f2","f3","f4");
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+
+ }
+ case 20: {
+ /*
+ * Non-POJO test to verify that full-tuple keys are working.
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env);
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env);
+ DataSet<Tuple2<Tuple2<Tuple2<Integer, Integer>, String>, Tuple2<Tuple2<Integer, Integer>, String> >> joinDs =
+ ds1.join(ds2).where(0).equalTo("f0.f0", "f0.f1"); // key is now Tuple2<Integer, Integer>
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "((1,1),one),((1,1),one)\n" +
+ "((2,2),two),((2,2),two)\n" +
+ "((3,3),three),((3,3),three)\n";
+
+ }
+ case 21: {
+ /*
+ * Non-POJO test to verify "nested" tuple-element selection.
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds1 = CollectionDataSets.getSmallNestedTupleDataSet(env);
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds2 = CollectionDataSets.getSmallNestedTupleDataSet(env);
+ DataSet<Tuple2<Tuple2<Tuple2<Integer, Integer>, String>, Tuple2<Tuple2<Integer, Integer>, String> >> joinDs =
+ ds1.join(ds2).where("f0.f0").equalTo("f0.f0"); // key is now Integer from Tuple2<Integer, Integer>
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "((1,1),one),((1,1),one)\n" +
+ "((2,2),two),((2,2),two)\n" +
+ "((3,3),three),((3,3),three)\n";
+
+ }
+ case 22: {
+ /*
+ * full pojo with full tuple
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds1 = CollectionDataSets.getSmallPojoDataSet(env);
+ DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedPojoMatchingDataSet(env);
+ DataSet<Tuple2<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long> >> joinDs =
+ ds1.join(ds2).where("*").equalTo("*");
+
+ joinDs.writeAsCsv(resultPath);
+ env.setDegreeOfParallelism(1);
+ env.execute();
+
+ // return expected result
+ return "1 First (10,100,1000,One) 10000,(1,First,10,100,1000,One,10000)\n" +
+ "2 Second (20,200,2000,Two) 20000,(2,Second,20,200,2000,Two,20000)\n" +
+ "3 Third (30,300,3000,Three) 30000,(3,Third,30,300,3000,Three,30000)\n";
+ }
default:
- throw new IllegalArgumentException("Invalid program id");
+ throw new IllegalArgumentException("Invalid program id: "+progId);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
index afefef9..bf1d404 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
@@ -45,7 +45,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class PartitionITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 1;
+ private static int NUM_PROGRAMS = 3;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -111,7 +111,7 @@ public class PartitionITCase extends JavaProgramTestBase {
"5\n" +
"6\n";
}
- case 2: {
+ case 1: {
/*
* Test hash partition by key selector
*/
@@ -141,7 +141,7 @@ public class PartitionITCase extends JavaProgramTestBase {
"5\n" +
"6\n";
}
- case 1: {
+ case 2: {
/*
* Test forced rebalancing
*/
@@ -200,7 +200,7 @@ public class PartitionITCase extends JavaProgramTestBase {
// return expected result
return result.toString();
}
- case 4: {
+ case 3: {
/*
* Test hash partition by key field and different DOP
*/
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
index 10ea882..a1957f9 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/ReduceITCase.java
@@ -42,7 +42,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class ReduceITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 9;
+ private static int NUM_PROGRAMS = 10;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -305,6 +305,33 @@ public class ReduceITCase extends JavaProgramTestBase {
"5,29,0,P-),2\n" +
"5,25,0,P-),3\n";
}
+ case 10: {
+ /*
+ * Case 2 with String-based field expression
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds.
+ groupBy("f4","f0").reduce(new Tuple5Reduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,0,Hallo,1\n" +
+ "2,3,2,Hallo Welt wie,1\n" +
+ "2,2,1,Hallo Welt,2\n" +
+ "3,9,0,P-),2\n" +
+ "3,6,5,BCD,3\n" +
+ "4,17,0,P-),1\n" +
+ "4,17,0,P-),2\n" +
+ "5,11,10,GHI,1\n" +
+ "5,29,0,P-),2\n" +
+ "5,25,0,P-),3\n";
+ }
+
default:
throw new IllegalArgumentException("Invalid program id");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
index 3ca8f31..b657545 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
@@ -24,11 +24,14 @@ import java.util.Collections;
import java.util.List;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple7;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.hadoop.io.IntWritable;
/**
* #######################################################################################################
@@ -136,6 +139,22 @@ public class CollectionDataSets {
return env.fromCollection(data, type);
}
+ public static DataSet<Tuple2<Tuple2<Integer, Integer>, String>> getSmallNestedTupleDataSet(ExecutionEnvironment env) {
+
+ List<Tuple2<Tuple2<Integer, Integer>, String>> data = new ArrayList<Tuple2<Tuple2<Integer, Integer>, String>>();
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,1), "one"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,2), "two"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,3), "three"));
+
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>>(
+ new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO,BasicTypeInfo.INT_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
+ return env.fromCollection(data, type);
+ }
+
public static DataSet<String> getStringDataSet(ExecutionEnvironment env) {
List<String> data = new ArrayList<String>();
@@ -241,7 +260,150 @@ public class CollectionDataSets {
public String toString() {
return myInt+","+myLong+","+myString;
}
-
+ }
+
+ public static DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> getSmallTuplebasedPojoMatchingDataSet(ExecutionEnvironment env) {
+ List<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> data = new ArrayList<Tuple7<Integer, String, Integer, Integer, Long, String, Long>>();
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(2, "Second",20, 200, 2000L, "Two", 20000L));
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(3, "Third",30, 300, 3000L, "Three", 30000L));
+ return env.fromCollection(data);
+ }
+
+ public static DataSet<POJO> getSmallPojoDataSet(ExecutionEnvironment env) {
+ List<POJO> data = new ArrayList<POJO>();
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(2, "Second",20, 200, 2000L, "Two", 20000L));
+ data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L));
+ return env.fromCollection(data);
+ }
+
+ public static class POJO {
+ public int number;
+ public String str;
+ public Tuple2<Integer, CustomType> nestedTupleWithCustom;
+ public NestedPojo nestedPojo;
+ public transient Long ignoreMe;
+ public POJO(int i0, String s0,
+ int i1, int i2, long l0, String s1,
+ long l1) {
+ this.number = i0;
+ this.str = s0;
+ this.nestedTupleWithCustom = new Tuple2<Integer, CustomType>(i1, new CustomType(i2, l0, s1));
+ this.nestedPojo = new NestedPojo();
+ this.nestedPojo.longNumber = l1;
+ }
+ public POJO() {}
+ @Override
+ public String toString() {
+ return number+" "+str+" "+nestedTupleWithCustom+" "+nestedPojo.longNumber;
+ }
+ }
+
+ public static class NestedPojo {
+ public static Object ignoreMe;
+ public long longNumber;
+ public NestedPojo() {}
+ }
+
+ public static DataSet<CrazyNested> getCrazyNestedDataSet(ExecutionEnvironment env) {
+ List<CrazyNested> data = new ArrayList<CrazyNested>();
+ data.add(new CrazyNested("aa"));
+ data.add(new CrazyNested("bb"));
+ data.add(new CrazyNested("bb"));
+ data.add(new CrazyNested("cc"));
+ data.add(new CrazyNested("cc"));
+ data.add(new CrazyNested("cc"));
+ return env.fromCollection(data);
+ }
+
+ public static class CrazyNested {
+ public CrazyNestedL1 nest_Lvl1;
+ public Long something; // test proper null-value handling
+ public CrazyNested() {}
+ public CrazyNested(String set, String second, long s) { // additional CTor to set all fields to non-null values
+ this(set);
+ something = s;
+ nest_Lvl1.a = second;
+ }
+ public CrazyNested(String set) {
+ nest_Lvl1 = new CrazyNestedL1();
+ nest_Lvl1.nest_Lvl2 = new CrazyNestedL2();
+ nest_Lvl1.nest_Lvl2.nest_Lvl3 = new CrazyNestedL3();
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4 = new CrazyNestedL4();
+ nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal = set;
+ }
+ }
+ public static class CrazyNestedL1 {
+ public String a;
+ public int b;
+ public CrazyNestedL2 nest_Lvl2;
+ }
+ public static class CrazyNestedL2 {
+ public CrazyNestedL3 nest_Lvl3;
+ }
+ public static class CrazyNestedL3 {
+ public CrazyNestedL4 nest_Lvl4;
+ }
+ public static class CrazyNestedL4 {
+ public String f1nal;
+ }
+
+ // Copied from TypeExtractorTest
+ public static class FromTuple extends Tuple3<String, String, Long> {
+ private static final long serialVersionUID = 1L;
+ public int special;
+ }
+
+ public static class FromTupleWithCTor extends FromTuple {
+ public FromTupleWithCTor() {}
+ public FromTupleWithCTor(int special, long tupleField ) {
+ this.special = special;
+ this.setField(tupleField, 2);
+ }
+ }
+ public static DataSet<FromTupleWithCTor> getPojoExtendingFromTuple(ExecutionEnvironment env) {
+ List<FromTupleWithCTor> data = new ArrayList<FromTupleWithCTor>();
+ data.add(new FromTupleWithCTor(1, 10L)); // 3x
+ data.add(new FromTupleWithCTor(1, 10L));
+ data.add(new FromTupleWithCTor(1, 10L));
+ data.add(new FromTupleWithCTor(2, 20L)); // 2x
+ data.add(new FromTupleWithCTor(2, 20L));
+ return env.fromCollection(data);
+ }
+
+ public static class PojoContainingTupleAndWritable {
+ public int someInt;
+ public String someString;
+ public IntWritable hadoopFan;
+ public Tuple2<Long, Long> theTuple;
+ public PojoContainingTupleAndWritable() {}
+ public PojoContainingTupleAndWritable(int i, long l1, long l2) {
+ hadoopFan = new IntWritable(i);
+ someInt = i;
+ theTuple = new Tuple2<Long, Long>(l1, l2);
+ }
+ }
+
+ public static DataSet<PojoContainingTupleAndWritable> getPojoContainingTupleAndWritable(ExecutionEnvironment env) {
+ List<PojoContainingTupleAndWritable> data = new ArrayList<PojoContainingTupleAndWritable>();
+ data.add(new PojoContainingTupleAndWritable(1, 10L, 100L)); // 1x
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L)); // 5x
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
+ return env.fromCollection(data);
+ }
+
+ public static DataSet<Tuple3<Integer,CrazyNested, POJO>> getTupleContainingPojos(ExecutionEnvironment env) {
+ List<Tuple3<Integer,CrazyNested, POJO>> data = new ArrayList<Tuple3<Integer,CrazyNested, POJO>>();
+ data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) )); // 3x
+ data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) ));
+ data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) ));
+ // POJO is not initialized according to the first two fields.
+ data.add(new Tuple3<Integer,CrazyNested, POJO>(2, new CrazyNested("two", "duo", 2L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) )); // 1x
+ return env.fromCollection(data);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
index 6dedcc1..16267f6 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recordJobs/kmeans/udfs/ComputeDistance.java
@@ -25,7 +25,6 @@ import org.apache.flink.api.java.record.functions.FunctionAnnotation.ConstantFie
import org.apache.flink.types.DoubleValue;
import org.apache.flink.types.IntValue;
import org.apache.flink.types.Record;
-import org.apache.flink.util.Collector;
/**
* Cross PACT computes the distance of all data points to all cluster
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
index 1925a94..0821af5 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/util/testjar/KMeansForTest.java
@@ -24,17 +24,20 @@ import java.util.Collection;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.Program;
-import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.functions.RichReduceFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
-
+import org.apache.flink.test.localDistributed.PackagedProgramEndToEndITCase;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.IterativeDataSet;
+/**
+ * This class belongs to the @see {@link PackagedProgramEndToEndITCase} test
+ *
+ */
@SuppressWarnings("serial")
public class KMeansForTest implements Program {
@@ -80,12 +83,8 @@ public class KMeansForTest implements Program {
.map(new SelectNearestCenter()).withBroadcastSet(loop, "centroids")
// count and sum point coordinates for each centroid
.map(new CountAppender())
- .groupBy(new KeySelector<DummyTuple3IntPointLong, Integer>() {
- @Override
- public Integer getKey(DummyTuple3IntPointLong value) throws Exception {
- return value.f0;
- }
- }).reduce(new CentroidAccumulator())
+ // !test if key expressions are working!
+ .groupBy("field0").reduce(new CentroidAccumulator())
// compute new centroids from point counts and coordinate sums
.map(new CentroidAverager());
@@ -228,16 +227,16 @@ public class KMeansForTest implements Program {
// Use this so that we can check whether POJOs and the POJO comparator also work
public static final class DummyTuple3IntPointLong {
- public Integer f0;
- public Point f1;
- public Long f2;
+ public Integer field0;
+ public Point field1;
+ public Long field2;
public DummyTuple3IntPointLong() {}
DummyTuple3IntPointLong(Integer f0, Point f1, Long f2) {
- this.f0 = f0;
- this.f1 = f1;
- this.f2 = f2;
+ this.field0 = f0;
+ this.field1 = f1;
+ this.field2 = f2;
}
}
@@ -255,7 +254,7 @@ public class KMeansForTest implements Program {
@Override
public DummyTuple3IntPointLong reduce(DummyTuple3IntPointLong val1, DummyTuple3IntPointLong val2) {
- return new DummyTuple3IntPointLong(val1.f0, val1.f1.add(val2.f1), val1.f2 + val2.f2);
+ return new DummyTuple3IntPointLong(val1.field0, val1.field1.add(val2.field1), val1.field2 + val2.field2);
}
}
@@ -264,7 +263,7 @@ public class KMeansForTest implements Program {
@Override
public Centroid map(DummyTuple3IntPointLong value) {
- return new Centroid(value.f0, value.f1.div(value.f2));
+ return new Centroid(value.field0, value.field1.div(value.field2));
}
}
}
[11/11] git commit: Add Pojo support to Scala API
Posted by rm...@apache.org.
Add Pojo support to Scala API
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/6b493fb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/6b493fb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/6b493fb0
Branch: refs/heads/master
Commit: 6b493fb080d4a589396c1754f5d89ba802c828d1
Parents: aca6fbc
Author: Robert Metzger <rm...@apache.org>
Authored: Mon Oct 6 11:45:24 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
.../api/common/typeutils/CompositeType.java | 1 +
.../flink/api/java/typeutils/TupleTypeInfo.java | 3 -
.../flink/api/java/typeutils/TypeExtractor.java | 23 ++--
.../java/typeutils/runtime/PojoComparator.java | 2 +-
.../java/type/extractor/TypeExtractorTest.java | 2 +-
.../typeutils/runtime/PojoComparatorTest.java | 2 +-
.../org/apache/flink/api/scala/DataSet.scala | 4 +-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 78 ++++--------
.../api/scala/operators/ExamplesITCase.scala | 127 +++++++++++++++++++
9 files changed, 170 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
index 60e9eab..1522ed1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
@@ -117,6 +117,7 @@ public abstract class CompositeType<T> extends TypeInformation<T> {
public int getPosition() {
return keyPosition;
}
+
public TypeInformation<?> getType() {
return type;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
index 82f9c50..177f033 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
@@ -62,14 +62,12 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
/**
* Comparator creation
*/
- private TypeSerializer<?>[] fieldSerializers;
private TypeComparator<?>[] fieldComparators;
private int[] logicalKeyFields;
private int comparatorHelperIndex = 0;
@Override
protected void initializeNewComparator(int localKeyCount) {
- fieldSerializers = new TypeSerializer[localKeyCount];
fieldComparators = new TypeComparator<?>[localKeyCount];
logicalKeyFields = new int[localKeyCount];
comparatorHelperIndex = 0;
@@ -78,7 +76,6 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
@Override
protected void addCompareField(int fieldId, TypeComparator<?> comparator) {
fieldComparators[comparatorHelperIndex] = comparator;
- fieldSerializers[comparatorHelperIndex] = types[fieldId].createSerializer();
logicalKeyFields[comparatorHelperIndex] = fieldId;
comparatorHelperIndex++;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 6231a74..5d216e9 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -398,7 +398,7 @@ public class TypeExtractor {
return ObjectArrayTypeInfo.getInfoFor(t, componentInfo);
}
// objects with generics are treated as raw type
- else if (t instanceof ParameterizedType) {
+ else if (t instanceof ParameterizedType) { //TODO
return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), typeHierarchy);
}
// no tuple, no TypeVariable, no generic type
@@ -936,14 +936,13 @@ public class TypeExtractor {
return pojoType;
}
-
// return a generic type
return new GenericTypeInfo<X>(clazz);
}
/**
* Checks if the given field is a valid pojo field:
- * - it is public
+ * - it is public
* OR
* - there are getter and setter methods for the field.
*
@@ -968,8 +967,8 @@ public class TypeExtractor {
for(Method m : clazz.getMethods()) {
// check for getter
- if( // The name should be "get<FieldName>".
- m.getName().toLowerCase().contains("get"+fieldNameLow) &&
+ if( // The name should be "get<FieldName>" or "<fieldName>" (for scala).
+ (m.getName().toLowerCase().contains("get"+fieldNameLow) || m.getName().toLowerCase().contains(fieldNameLow)) &&
// no arguments for the getter
m.getParameterTypes().length == 0 &&
// return type is same as field type (or the generic variant of it)
@@ -980,12 +979,12 @@ public class TypeExtractor {
}
hasGetter = true;
}
- // check for setters
- if( m.getName().toLowerCase().contains("set"+fieldNameLow) &&
- m.getParameterTypes().length == 1 && // one parameter of the field's type
- ( m.getParameterTypes()[0].equals( fieldType ) || (fieldTypeGeneric != null && m.getGenericParameterTypes()[0].equals(fieldTypeGeneric) ) )&&
- // return type is void.
- m.getReturnType().equals(Void.TYPE)
+ // check for setters (<FieldName>_$eq for scala)
+ if((m.getName().toLowerCase().contains("set"+fieldNameLow) || m.getName().toLowerCase().contains(fieldNameLow+"_$eq")) &&
+ m.getParameterTypes().length == 1 && // one parameter of the field's type
+ ( m.getParameterTypes()[0].equals( fieldType ) || (fieldTypeGeneric != null && m.getGenericParameterTypes()[0].equals(fieldTypeGeneric) ) )&&
+ // return type is void.
+ m.getReturnType().equals(Void.TYPE)
) {
if(hasSetter) {
throw new IllegalStateException("Detected more than one getters");
@@ -993,7 +992,7 @@ public class TypeExtractor {
hasSetter = true;
}
}
- if( hasGetter && hasSetter) {
+ if(hasGetter && hasSetter) {
return true;
} else {
if(!hasGetter) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
index 9d7eed4..51d8090 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
@@ -340,7 +340,7 @@ public final class PojoComparator<T> extends CompositeTypeComparator<T> implemen
public int extractKeys(Object record, Object[] target, int index) {
int localIndex = index;
for (int i = 0; i < comparators.length; i++) {
- if(comparators[i] instanceof PojoComparator || comparators[i] instanceof TupleComparator) {
+ if(comparators[i] instanceof CompositeTypeComparator) {
localIndex += comparators[i].extractKeys(accessField(keyFields[i], record), target, localIndex) -1;
} else {
// non-composite case (= atomic). We can assume this to have only one key.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index a092a53..b4b1c19 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -1247,7 +1247,7 @@ public class TypeExtractorTest {
public static class InType extends MyObject<String> {}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
- @Ignore
+// @Ignore
public void testParamertizedCustomObject() {
RichMapFunction<?, ?> function = new RichMapFunction<InType, MyObject<String>>() {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
index 5cb31ca..61f6167 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
@@ -31,7 +31,7 @@ import org.junit.Assert;
import org.junit.Ignore;
-@Ignore // TODO
+//@Ignore // TODO
public class PojoComparatorTest extends ComparatorTestBase<PojoContainingTuple> {
TypeInformation<PojoContainingTuple> type = TypeExtractor.getForClass(PojoContainingTuple.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
index 28624bc..895b964 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
@@ -615,11 +615,11 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
* This only works on CaseClass DataSets.
*/
def groupBy(firstField: String, otherFields: String*): GroupedDataSet[T] = {
- val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
+ // val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
new GroupedDataSet[T](
this,
- new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType,false))
+ new Keys.ExpressionKeys[T](firstField +: otherFields.toArray, javaSet.getType))
}
// public UnsortedGrouping<T> groupBy(String... fields) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
index c9a3bbf..3e9d4c6 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
@@ -32,68 +32,42 @@ abstract class CaseClassTypeInfo[T <: Product](
val fieldNames: Seq[String])
extends TupleTypeInfoBase[T](clazz, fieldTypes: _*) {
- override def createComparator(logicalKeyFields: Array[Int],
- orders: Array[Boolean], offset: Int): TypeComparator[T] = {
- // sanity checks
- if (logicalKeyFields == null || orders == null
- || logicalKeyFields.length != orders.length || logicalKeyFields.length > types.length) {
- throw new IllegalArgumentException
- }
-
- // No special handling of leading Key field as in JavaTupleComparator for now
-
- // --- general case ---
- var maxKey: Int = -1
+ def getFieldIndices(fields: Array[String]): Array[Int] = {
+ fields map { x => fieldNames.indexOf(x) }
+ }
- for (key <- logicalKeyFields) {
- maxKey = Math.max(key, maxKey)
- }
+ /*
+ * Comparator construction
+ */
+ var fieldComparators: Array[TypeComparator[_]] = null
+ var logicalKeyFields : Array[Int] = null
+ var comparatorHelperIndex = 0
- if (maxKey >= types.length) {
- throw new IllegalArgumentException("The key position " + maxKey + " is out of range for " +
- "Tuple" + types.length)
- }
+ override protected def initializeNewComparator(localKeyCount: Int): Unit = {
+ fieldComparators = new Array(localKeyCount)
+ logicalKeyFields = new Array(localKeyCount)
+ comparatorHelperIndex = 0
+ }
- // create the comparators for the individual fields
- val fieldComparators: Array[TypeComparator[_]] = new Array(logicalKeyFields.length)
+ override protected def addCompareField(fieldId: Int, comparator: TypeComparator[_]): Unit = {
+ fieldComparators(comparatorHelperIndex) = comparator
+ logicalKeyFields(comparatorHelperIndex) = fieldId
+ comparatorHelperIndex += 1
+ }
- for (i <- 0 until logicalKeyFields.length) {
- val keyPos = logicalKeyFields(i)
- if (types(keyPos).isKeyType && types(keyPos).isInstanceOf[AtomicType[_]]) {
- fieldComparators(i) = types(keyPos).asInstanceOf[AtomicType[_]].createComparator(orders(i))
- } else {
- throw new IllegalArgumentException(
- "The field at position " + i + " (" + types(keyPos) + ") is no atomic key type.")
- }
+ override protected def getNewComparator: TypeComparator[T] = {
+ val finalLogicalKeyFields = logicalKeyFields.take(comparatorHelperIndex)
+ val finalComparators = fieldComparators.take(comparatorHelperIndex)
+ var maxKey: Int = 0
+ for (key <- finalLogicalKeyFields) {
+ maxKey = Math.max(maxKey, key)
}
-
- // create the serializers for the prefix up to highest key position
val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](maxKey + 1)
for (i <- 0 to maxKey) {
fieldSerializers(i) = types(i).createSerializer
}
-
- new CaseClassComparator[T](logicalKeyFields, fieldComparators, fieldSerializers)
- }
-
- def getFieldIndices(fields: Array[String]): Array[Int] = {
- fields map { x => fieldNames.indexOf(x) }
- }
-
- override protected def initializeNewComparator(localKeyCount: Int): Unit = {
- throw new UnsupportedOperationException("The Scala API is not using the composite " +
- "type comparator creation")
- }
-
- override protected def getNewComparator: TypeComparator[T] = {
- throw new UnsupportedOperationException("The Scala API is not using the composite " +
- "type comparator creation")
- }
-
- override protected def addCompareField(fieldId: Int, comparator: TypeComparator[_]): Unit = {
- throw new UnsupportedOperationException("The Scala API is not using the composite " +
- "type comparator creation")
+ new CaseClassComparator[T](finalLogicalKeyFields, finalComparators, fieldSerializers)
}
override def toString = clazz.getSimpleName + "(" + fieldNames.zip(types).map {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6b493fb0/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
new file mode 100644
index 0000000..d5ae6b6
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.apache.flink.api.scala._
+import org.junit.runners.Parameterized.Parameters
+import scala.collection.JavaConverters._
+
+import scala.collection.mutable
+
+// TODO case class Tuple2[T1, T2](_1: T1, _2: T2)
+// TODO case class Foo(a: Int, b: String)
+
+class Nested(var myLong: Long) {
+ def this() = {
+ this(0);
+ }
+}
+class Pojo(var myString: String, var myInt: Int, myLong: Long) {
+ var nested = new Nested(myLong)
+
+ def this() = {
+ this("", 0, 0)
+ }
+
+ override def toString() = "myString="+myString+" myInt="+myInt+" nested.myLong="+nested.myLong
+}
+
+object ExampleProgs {
+ var NUM_PROGRAMS: Int = 3
+
+ def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
+ progId match {
+ case 1 =>
+ /*
+ Test nested tuples with int offset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
+
+ val grouped = ds.groupBy(0).reduce( { (e1, e2) => ((e1._1._1,e1._1._2), e1._2+e2._2)})
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "((this,hello),3)\n((this,is),3)\n"
+ case 2 =>
+ /*
+ Test nested tuples with int offset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
+
+ val grouped = ds.groupBy("f0.f0").reduce( { (e1, e2) => ((e1._1._1,e1._1._2), e1._2+e2._2)})
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "((this,is),6)\n"
+ case 3 =>
+ /*
+ Test nested pojos
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( new Pojo("one", 1, 1L),new Pojo("one", 1, 1L),new Pojo("two", 666, 2L) )
+
+ val grouped = ds.groupBy("nested.myLong").reduce {
+ (p1, p2) =>
+ p1.myInt += p2.myInt
+ p1
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
+ }
+ }
+}
+
+@RunWith(classOf[Parameterized])
+class ExamplesITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = ExampleProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object ExamplesITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to ExampleProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
\ No newline at end of file
[08/11] git commit: [FLINK-1032] Rework support for POJO types in the
Java API
Posted by rm...@apache.org.
[FLINK-1032] Rework support for POJO types in the Java API
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/926f835a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/926f835a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/926f835a
Branch: refs/heads/master
Commit: 926f835adb9cb77bcda19e4ed009cc07c89cea74
Parents: ec82d97
Author: Robert Metzger <rm...@apache.org>
Authored: Mon Aug 11 13:32:45 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
.../io/AvroInputFormatTypeExtractionTest.java | 6 +-
.../api/java/io/jdbc/JDBCOutputFormatTest.java | 1 -
.../compiler/postpass/JavaApiPostPass.java | 9 +-
.../WorksetIterationsJavaApiCompilerTest.java | 6 +-
.../api/common/InvalidProgramException.java | 4 +
.../common/functions/InvalidTypesException.java | 4 +
.../common/operators/CollectionExecutor.java | 4 +-
.../operators/base/CoGroupOperatorBase.java | 5 +-
.../operators/base/GroupReduceOperatorBase.java | 27 +-
.../common/operators/base/JoinOperatorBase.java | 6 +-
.../operators/base/ReduceOperatorBase.java | 5 +-
.../api/common/typeinfo/BasicArrayTypeInfo.java | 5 +
.../api/common/typeinfo/BasicTypeInfo.java | 5 +
.../api/common/typeinfo/CompositeType.java | 30 --
.../api/common/typeinfo/NothingTypeInfo.java | 5 +
.../common/typeinfo/PrimitiveArrayTypeInfo.java | 5 +
.../api/common/typeinfo/TypeInformation.java | 6 +
.../api/common/typeutils/CompositeType.java | 129 +++++
.../typeutils/CompositeTypeComparator.java | 36 ++
.../common/typeutils/GenericPairComparator.java | 18 +-
.../api/common/typeutils/TypeComparator.java | 5 +-
.../common/typeutils/TypePairComparator.java | 1 -
.../typeutils/base/BasicTypeComparator.java | 14 +-
.../typeutils/record/RecordComparator.java | 5 +-
.../types/KeyFieldOutOfBoundsException.java | 5 +
flink-examples/flink-java-examples/pom.xml | 24 +
.../CollectionExecutionExample.java | 48 --
.../RemoteCollectorOutputFormatExample.java | 114 -----
.../CollectionExecutionExample.java | 48 ++
.../RemoteCollectorOutputFormatExample.java | 114 +++++
.../examples/java/wordcount/PojoExample.java | 171 +++++++
.../java/org/apache/flink/api/java/DataSet.java | 22 +-
.../flink/api/java/ExecutionEnvironment.java | 1 +
.../flink/api/java/SerializationSpeedTest.java | 150 ++++++
.../api/java/operators/AggregateOperator.java | 2 +-
.../api/java/operators/CoGroupOperator.java | 57 ++-
.../api/java/operators/DistinctOperator.java | 6 +-
.../api/java/operators/GroupReduceOperator.java | 29 +-
.../flink/api/java/operators/JoinOperator.java | 59 ++-
.../apache/flink/api/java/operators/Keys.java | 380 ++++++++------
.../api/java/operators/PartitionOperator.java | 4 +-
.../api/java/operators/ReduceOperator.java | 3 +-
.../api/java/typeutils/GenericTypeInfo.java | 8 +
.../api/java/typeutils/ObjectArrayTypeInfo.java | 5 +
.../flink/api/java/typeutils/PojoField.java | 9 +-
.../flink/api/java/typeutils/PojoTypeInfo.java | 192 ++++---
.../api/java/typeutils/RecordTypeInfo.java | 5 +
.../flink/api/java/typeutils/TupleTypeInfo.java | 108 ++--
.../api/java/typeutils/TupleTypeInfoBase.java | 111 +++-
.../flink/api/java/typeutils/TypeExtractor.java | 237 +++++++--
.../api/java/typeutils/TypeInfoParser.java | 2 +-
.../flink/api/java/typeutils/ValueTypeInfo.java | 5 +
.../api/java/typeutils/WritableTypeInfo.java | 19 +
.../runtime/CopyableValueComparator.java | 10 +-
.../runtime/GenericTypeComparator.java | 13 +-
.../java/typeutils/runtime/PojoComparator.java | 183 +++----
.../java/typeutils/runtime/PojoSerializer.java | 45 +-
.../runtime/RuntimePairComparatorFactory.java | 2 -
.../java/typeutils/runtime/TupleComparator.java | 30 +-
.../typeutils/runtime/TupleComparatorBase.java | 32 +-
.../java/typeutils/runtime/ValueComparator.java | 12 +-
.../typeutils/runtime/WritableComparator.java | 12 +-
.../api/java/operator/CoGroupOperatorTest.java | 86 ++--
.../flink/api/java/operator/GroupingTest.java | 46 +-
.../api/java/operator/JoinOperatorTest.java | 376 +++++++++++++-
.../flink/api/java/operators/KeysTest.java | 186 +++++++
.../type/extractor/PojoTypeInformationTest.java | 81 ++-
.../java/type/extractor/TypeExtractorTest.java | 511 ++++++++++++++++++-
.../runtime/GenericPairComparatorTest.java | 3 +-
.../typeutils/runtime/PojoSerializerTest.java | 82 ++-
.../typeutils/runtime/TupleSerializerTest.java | 3 +-
.../drivers/GroupReduceDriverTest.java | 10 +-
.../drivers/ReduceCombineDriverTest.java | 10 +-
.../operators/drivers/ReduceDriverTest.java | 10 +-
.../sort/MassiveStringSortingITCase.java | 2 +-
.../testutils/types/IntListComparator.java | 10 +-
.../testutils/types/IntPairComparator.java | 11 +-
.../testutils/types/StringPairComparator.java | 10 +-
.../operators/util/OutputEmitterTest.java | 10 +-
.../scala/operators/ScalaAggregateOperator.java | 2 +-
.../org/apache/flink/api/scala/DataSet.scala | 18 +-
.../apache/flink/api/scala/crossDataSet.scala | 2 +-
.../scala/typeutils/CaseClassComparator.scala | 6 +-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 18 +-
.../api/scala/unfinishedKeyPairOperation.scala | 16 +-
.../scala/operators/CoGroupOperatorTest.scala | 13 +-
.../api/scala/operators/JoinOperatorTest.scala | 13 +-
.../scala/runtime/TupleComparatorILD2Test.scala | 2 +-
.../scala/runtime/TupleComparatorILD3Test.scala | 2 +-
.../runtime/TupleComparatorILDC3Test.scala | 2 +-
.../runtime/TupleComparatorILDX1Test.scala | 2 +-
.../runtime/TupleComparatorILDXC2Test.scala | 2 +-
.../scala/runtime/TupleComparatorISD1Test.scala | 2 +-
.../scala/runtime/TupleComparatorISD2Test.scala | 2 +-
.../scala/runtime/TupleComparatorISD3Test.scala | 2 +-
.../WordCountNestedPOJOITCase.java | 118 +++++
.../WordCountPOJOITCase.java | 48 --
.../WordCountSimplePOJOITCase.java | 102 ++++
.../EnumTriangleOptITCase.java | 2 +-
.../exampleScalaPrograms/PageRankITCase.java | 12 +-
.../VertexWithAdjacencyListComparator.java | 10 +-
.../VertexWithRankAndDanglingComparator.java | 10 +-
.../types/VertexWithRankComparator.java | 10 +-
.../test/javaApiOperators/CoGroupITCase.java | 151 +++++-
.../javaApiOperators/GroupReduceITCase.java | 119 ++++-
.../flink/test/javaApiOperators/JoinITCase.java | 173 ++++++-
.../test/javaApiOperators/PartitionITCase.java | 8 +-
.../test/javaApiOperators/ReduceITCase.java | 29 +-
.../util/CollectionDataSets.java | 164 +++++-
.../recordJobs/kmeans/udfs/ComputeDistance.java | 1 -
.../flink/test/util/testjar/KMeansForTest.java | 31 +-
111 files changed, 4025 insertions(+), 1132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
index fe3b6c8..23fbab3 100644
--- a/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
+++ b/flink-addons/flink-avro/src/test/java/org/apache/flink/api/java/io/AvroInputFormatTypeExtractionTest.java
@@ -23,7 +23,7 @@ import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.core.fs.Path;
import org.junit.Assert;
@@ -43,8 +43,8 @@ public class AvroInputFormatTypeExtractionTest {
TypeInformation<?> typeInfoDataSet = input.getType();
- Assert.assertTrue(typeInfoDirect instanceof GenericTypeInfo);
- Assert.assertTrue(typeInfoDataSet instanceof GenericTypeInfo);
+ Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
+ Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
index 5d24fad..7d004f9 100644
--- a/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
+++ b/flink-addons/flink-jdbc/src/test/java/org/apache/flink/api/java/io/jdbc/JDBCOutputFormatTest.java
@@ -26,7 +26,6 @@ import java.sql.Statement;
import org.junit.Assert;
-import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.junit.After;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-compiler/src/main/java/org/apache/flink/compiler/postpass/JavaApiPostPass.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/main/java/org/apache/flink/compiler/postpass/JavaApiPostPass.java b/flink-compiler/src/main/java/org/apache/flink/compiler/postpass/JavaApiPostPass.java
index 4756bc9..acce51f 100644
--- a/flink-compiler/src/main/java/org/apache/flink/compiler/postpass/JavaApiPostPass.java
+++ b/flink-compiler/src/main/java/org/apache/flink/compiler/postpass/JavaApiPostPass.java
@@ -31,8 +31,8 @@ import org.apache.flink.api.common.operators.base.DeltaIterationBase;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.api.common.typeinfo.AtomicType;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypePairComparatorFactory;
@@ -40,6 +40,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.operators.translation.PlanUnwrappingReduceGroupOperator;
import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.runtime.RuntimeComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory;
import org.apache.flink.api.java.typeutils.runtime.RuntimeStatefulSerializerFactory;
@@ -292,7 +293,7 @@ public class JavaApiPostPass implements OptimizerPostPass {
TypeComparator<T> comparator;
if (typeInfo instanceof CompositeType) {
- comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder);
+ comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0);
}
else if (typeInfo instanceof AtomicType) {
// handle grouping of atomic types
@@ -306,8 +307,8 @@ public class JavaApiPostPass implements OptimizerPostPass {
}
private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
- if (!(typeInfo1.isTupleType() && typeInfo2.isTupleType())) {
- throw new RuntimeException("The runtime currently supports only keyed binary operations on tuples.");
+ if (!(typeInfo1.isTupleType() || typeInfo1 instanceof PojoTypeInfo) && (typeInfo2.isTupleType() || typeInfo2 instanceof PojoTypeInfo)) {
+ throw new RuntimeException("The runtime currently supports only keyed binary operations (such as joins) on tuples and POJO types.");
}
// @SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
----------------------------------------------------------------------
diff --git a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
index d07743e..ef756d0 100644
--- a/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
+++ b/flink-compiler/src/test/java/org/apache/flink/compiler/WorksetIterationsJavaApiCompilerTest.java
@@ -78,7 +78,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
// verify joinWithSolutionSet
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
@@ -125,7 +125,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
// verify joinWithSolutionSet
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
@@ -170,7 +170,7 @@ public class WorksetIterationsJavaApiCompilerTest extends CompilerTestBase {
// verify joinWithSolutionSet
assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getInput1().getShipStrategy());
assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
- assertEquals(new FieldList(0, 1), joinWithSolutionSetNode.getKeysForInput1());
+ assertEquals(new FieldList(1, 0), joinWithSolutionSetNode.getKeysForInput1());
// verify reducer
assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/InvalidProgramException.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/InvalidProgramException.java b/flink-core/src/main/java/org/apache/flink/api/common/InvalidProgramException.java
index 73162ee..6753982 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/InvalidProgramException.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/InvalidProgramException.java
@@ -43,4 +43,8 @@ public class InvalidProgramException extends RuntimeException {
public InvalidProgramException(String message) {
super(message);
}
+
+ public InvalidProgramException(String message, Throwable e) {
+ super(message, e);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/functions/InvalidTypesException.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/InvalidTypesException.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/InvalidTypesException.java
index 0501505..611765e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/InvalidTypesException.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/InvalidTypesException.java
@@ -43,4 +43,8 @@ public class InvalidTypesException extends InvalidProgramException {
public InvalidTypesException(String message) {
super(message);
}
+
+ public InvalidTypesException(String message, Throwable e) {
+ super(message, e);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
index ac8e1f3..ea68554 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
@@ -45,8 +45,8 @@ import org.apache.flink.api.common.operators.base.DeltaIterationBase;
import org.apache.flink.api.common.operators.base.DeltaIterationBase.SolutionSetPlaceHolder;
import org.apache.flink.api.common.operators.base.DeltaIterationBase.WorksetPlaceHolder;
import org.apache.flink.api.common.operators.util.TypeComparable;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.types.Value;
import org.apache.flink.util.Visitor;
@@ -349,7 +349,7 @@ public class CollectionExecutor {
int[] keyColumns = iteration.getSolutionSetKeyFields();
boolean[] inputOrderings = new boolean[keyColumns.length];
- TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings);
+ TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0);
Map<TypeComparable<T>, T> solutionMap = new HashMap<TypeComparable<T>, T>(solutionInputData.size());
// fill the solution from the initial input
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
index 6aa3da0..bca909f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/CoGroupOperatorBase.java
@@ -31,8 +31,8 @@ import org.apache.flink.api.common.operators.util.ListKeyGroupedIterator;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.GenericPairComparator;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
@@ -227,13 +227,12 @@ public class CoGroupOperatorBase<IN1, IN2, OUT, FT extends CoGroupFunction<IN1,
return result;
}
- @SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
if (!(inputType instanceof CompositeType)) {
throw new InvalidProgramException("Input types of coGroup must be composite types.");
}
- return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections);
+ return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0);
}
private static class CoGroupSortListIterator<IN1, IN2> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
index 8a3bf65..f500717 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java
@@ -33,11 +33,13 @@ import org.apache.flink.api.common.operators.util.ListKeyGroupedIterator;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import com.google.common.base.Preconditions;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -154,15 +156,18 @@ public class GroupReduceOperatorBase<IN, OUT, FT extends GroupReduceFunction<IN,
}
if (inputType instanceof CompositeType) {
- @SuppressWarnings("unchecked")
- final TypeComparator<IN> sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings);
-
- Collections.sort(inputData, new Comparator<IN>() {
- @Override
- public int compare(IN o1, IN o2) {
- return sortComparator.compare(o1, o2);
- }
- });
+ if(sortColumns.length == 0) { // => all reduce. No comparator
+ Preconditions.checkArgument(sortOrderings.length == 0);
+ } else {
+ final TypeComparator<IN> sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0);
+
+ Collections.sort(inputData, new Comparator<IN>() {
+ @Override
+ public int compare(IN o1, IN o2) {
+ return sortComparator.compare(o1, o2);
+ }
+ });
+ }
}
FunctionUtils.setFunctionRuntimeContext(function, ctx);
@@ -188,7 +193,7 @@ public class GroupReduceOperatorBase<IN, OUT, FT extends GroupReduceFunction<IN,
} else {
final TypeSerializer<IN> inputSerializer = inputType.createSerializer();
boolean[] keyOrderings = new boolean[keyColumns.length];
- final TypeComparator<IN> comparator = ((CompositeType<IN>) inputType).createComparator(keyColumns, keyOrderings);
+ final TypeComparator<IN> comparator = ((CompositeType<IN>) inputType).createComparator(keyColumns, keyOrderings, 0);
ListKeyGroupedIterator<IN> keyedIterator = new ListKeyGroupedIterator<IN>(inputData, inputSerializer, comparator, mutableObjectSafeMode);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
index 3d5cf72..7bfe39f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/JoinOperatorBase.java
@@ -29,8 +29,8 @@ import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
import org.apache.flink.api.common.typeinfo.AtomicType;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.GenericPairComparator;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypePairComparator;
@@ -86,7 +86,7 @@ public class JoinOperatorBase<IN1, IN2, OUT, FT extends FlatJoinFunction<IN1, IN
boolean[] orders = new boolean[keyPositions.length];
Arrays.fill(orders, true);
- leftComparator = ((CompositeType<IN1>) leftInformation).createComparator(keyPositions, orders);
+ leftComparator = ((CompositeType<IN1>) leftInformation).createComparator(keyPositions, orders, 0);
}else{
throw new RuntimeException("Type information for left input of type " + leftInformation.getClass()
.getCanonicalName() + " is not supported. Could not generate a comparator.");
@@ -99,7 +99,7 @@ public class JoinOperatorBase<IN1, IN2, OUT, FT extends FlatJoinFunction<IN1, IN
boolean[] orders = new boolean[keyPositions.length];
Arrays.fill(orders, true);
- rightComparator = ((CompositeType<IN2>) rightInformation).createComparator(keyPositions, orders);
+ rightComparator = ((CompositeType<IN2>) rightInformation).createComparator(keyPositions, orders, 0);
}else{
throw new RuntimeException("Type information for right input of type " + rightInformation.getClass()
.getCanonicalName() + " is not supported. Could not generate a comparator.");
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
index dc01637..30ff176 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/ReduceOperatorBase.java
@@ -28,8 +28,8 @@ import org.apache.flink.api.common.operators.util.TypeComparable;
import org.apache.flink.api.common.operators.util.UserCodeClassWrapper;
import org.apache.flink.api.common.operators.util.UserCodeObjectWrapper;
import org.apache.flink.api.common.operators.util.UserCodeWrapper;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -125,7 +125,6 @@ public class ReduceOperatorBase<T, FT extends ReduceFunction<T>> extends SingleI
// --------------------------------------------------------------------------------------------
- @SuppressWarnings("unchecked")
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, boolean mutableObjectSafeMode) throws Exception {
// make sure we can handle empty inputs
@@ -151,7 +150,7 @@ public class ReduceOperatorBase<T, FT extends ReduceFunction<T>> extends SingleI
if (inputColumns.length > 0) {
boolean[] inputOrderings = new boolean[inputColumns.length];
- TypeComparator<T> inputComparator = ((CompositeType<T>) inputType).createComparator(inputColumns, inputOrderings);
+ TypeComparator<T> inputComparator = ((CompositeType<T>) inputType).createComparator(inputColumns, inputOrderings, 0);
Map<TypeComparable<T>, T> aggregateMap = new HashMap<TypeComparable<T>, T>(inputData.size() / 10);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
index bbf4c2a..646a549 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicArrayTypeInfo.java
@@ -68,6 +68,11 @@ public class BasicArrayTypeInfo<T, C> extends TypeInformation<T> {
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@Override
public Class<T> getTypeClass() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
index cb2247d..a152b4a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/BasicTypeInfo.java
@@ -92,6 +92,11 @@ public class BasicTypeInfo<T> extends TypeInformation<T> implements AtomicType<T
}
@Override
+ public int getTotalFields() {
+ return 1;
+ }
+
+ @Override
public Class<T> getTypeClass() {
return this.clazz;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/CompositeType.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/CompositeType.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/CompositeType.java
deleted file mode 100644
index 075b528..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/CompositeType.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.common.typeinfo;
-
-import org.apache.flink.api.common.typeutils.TypeComparator;
-
-
-/**
- *
- */
-public interface CompositeType<T> {
-
- TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders);
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NothingTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NothingTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NothingTypeInfo.java
index 5cd5db3..dba0e6f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NothingTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/NothingTypeInfo.java
@@ -39,6 +39,11 @@ public class NothingTypeInfo extends TypeInformation<Nothing> {
}
@Override
+ public int getTotalFields() {
+ return 0;
+ }
+
+ @Override
public Class<Nothing> getTypeClass() {
return Nothing.class;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
index 19ef663..5163801 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/PrimitiveArrayTypeInfo.java
@@ -69,6 +69,11 @@ public class PrimitiveArrayTypeInfo<T> extends TypeInformation<T> {
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@Override
public Class<T> getTypeClass() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/TypeInformation.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/TypeInformation.java b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/TypeInformation.java
index 6eb08b8..0f86486 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/TypeInformation.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeinfo/TypeInformation.java
@@ -33,4 +33,10 @@ public abstract class TypeInformation<T> {
public abstract boolean isKeyType();
public abstract TypeSerializer<T> createSerializer();
+
+ /**
+ * @return The number of fields in this type, including its sub-fields (for compsite types)
+ */
+ public abstract int getTotalFields();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
new file mode 100644
index 0000000..60e9eab
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.AtomicType;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+
+
+/**
+ * Type Information for Tuple and Pojo types
+ *
+ * The class is taking care of serialization and comparators for Tuples as well.
+ * See @see {@link Keys} class for fields setup.
+ */
+public abstract class CompositeType<T> extends TypeInformation<T> {
+
+ protected final Class<T> typeClass;
+
+ public CompositeType(Class<T> typeClass) {
+ this.typeClass = typeClass;
+ }
+
+ /**
+ * Returns the keyPosition for the given fieldPosition, offsetted by the given offset
+ */
+ public abstract void getKey(String fieldExpression, int offset, List<FlatFieldDescriptor> result);
+
+ public abstract <X> TypeInformation<X> getTypeAt(int pos);
+
+ /**
+ * Initializes the internal state inside a Composite type to create a new comparator
+ * (such as the lists / arrays for the fields and field comparators)
+ * @param localKeyCount
+ */
+ protected abstract void initializeNewComparator(int localKeyCount);
+
+ /**
+ * Add a field for comparison in this type.
+ */
+ protected abstract void addCompareField(int fieldId, TypeComparator<?> comparator);
+
+ /**
+ * Get the actual comparator we've initialized.
+ */
+ protected abstract TypeComparator<T> getNewComparator();
+
+
+ /**
+ * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
+ * to create the actual comparators
+ * @return
+ */
+ public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset) {
+ initializeNewComparator(logicalKeyFields.length);
+
+ for(int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
+ int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
+ int logicalField = logicalFieldOffset; // this is the global/logical field number
+ for(int localFieldId = 0; localFieldId < this.getArity(); localFieldId++) {
+ TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
+
+ if(localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
+ // we found an atomic key --> create comparator
+ addCompareField(localFieldId, ((AtomicType<?>) localFieldType).createComparator(orders[logicalKeyFieldIndex]) );
+ } else if(localFieldType instanceof CompositeType && // must be a composite type
+ ( logicalField <= logicalKeyField //check if keyField can be at or behind the current logicalField
+ && logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1) ) // check if logical field + lookahead could contain our key
+ ) {
+ // we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
+ addCompareField(localFieldId, ((CompositeType<?>) localFieldType).createComparator(new int[] {logicalKeyField}, orders, logicalField));
+ }
+
+ // maintain logicalField
+ if(localFieldType instanceof CompositeType) {
+ // we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
+ logicalField += localFieldType.getTotalFields() - 1;
+ }
+ logicalField++;
+ }
+ }
+ return getNewComparator();
+ }
+
+
+
+ public static class FlatFieldDescriptor {
+ private int keyPosition;
+ private TypeInformation<?> type;
+
+ public FlatFieldDescriptor(int keyPosition, TypeInformation<?> type) {
+ if( !(type instanceof AtomicType)) {
+ throw new IllegalArgumentException("A flattened field can only be an atomic type");
+ }
+ this.keyPosition = keyPosition;
+ this.type = type;
+ }
+
+
+ public int getPosition() {
+ return keyPosition;
+ }
+ public TypeInformation<?> getType() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return "FlatFieldDescriptor [position="+keyPosition+" typeInfo="+type+"]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeComparator.java
new file mode 100644
index 0000000..8323dc3
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeTypeComparator.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public abstract class CompositeTypeComparator<T> extends TypeComparator<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public TypeComparator[] getFlatComparators() {
+ List<TypeComparator> flatComparators = new LinkedList<TypeComparator>();
+ this.getFlatComparator(flatComparators);
+ return flatComparators.toArray(new TypeComparator[flatComparators.size()]);
+ }
+
+ public abstract void getFlatComparator(List<TypeComparator> flatComparators);
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericPairComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericPairComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericPairComparator.java
index b64a3a3..09d4aee 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericPairComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/GenericPairComparator.java
@@ -36,13 +36,15 @@ public class GenericPairComparator<T1, T2> extends TypePairComparator<T1, T2>
private final TypeComparator<Object>[] comparators2;
private final Object[] referenceKeyFields;
+
+ private final Object[] candidateKeyFields;
@SuppressWarnings("unchecked")
public GenericPairComparator(TypeComparator<T1> comparator1, TypeComparator<T2> comparator2) {
this.comparator1 = comparator1;
this.comparator2 = comparator2;
- this.comparators1 = comparator1.getComparators();
- this.comparators2 = comparator2.getComparators();
+ this.comparators1 = comparator1.getFlatComparators();
+ this.comparators2 = comparator2.getFlatComparators();
if(comparators1.length != comparators2.length) {
throw new IllegalArgumentException("Number of key fields and comparators differ.");
@@ -56,19 +58,19 @@ public class GenericPairComparator<T1, T2> extends TypePairComparator<T1, T2>
}
this.referenceKeyFields = new Object[numKeys];
+ this.candidateKeyFields = new Object[numKeys];
}
@Override
public void setReference(T1 reference) {
- Object[] keys = comparator1.extractKeys(reference);
- System.arraycopy(keys, 0, referenceKeyFields, 0, keys.length);
+ comparator1.extractKeys(reference, referenceKeyFields, 0);
}
@Override
public boolean equalToReference(T2 candidate) {
- Object[] keys = comparator2.extractKeys(candidate);
+ comparator2.extractKeys(candidate, candidateKeyFields, 0);
for (int i = 0; i < this.comparators1.length; i++) {
- if (this.comparators1[i].compare(referenceKeyFields[i], keys[i]) != 0) {
+ if (this.comparators1[i].compare(referenceKeyFields[i], candidateKeyFields[i]) != 0) {
return false;
}
}
@@ -77,11 +79,11 @@ public class GenericPairComparator<T1, T2> extends TypePairComparator<T1, T2>
@Override
public int compareToReference(T2 candidate) {
- Object[] keys = comparator2.extractKeys(candidate);
+ comparator2.extractKeys(candidate, candidateKeyFields, 0);
for (int i = 0; i < this.comparators1.length; i++) {
// We reverse ordering here because our "compareToReference" does work in a mirrored
// way compared to Comparable.compareTo
- int res = this.comparators1[i].compare(keys[i], referenceKeyFields[i]);
+ int res = this.comparators1[i].compare(candidateKeyFields[i], referenceKeyFields[i]);
if(res != 0) {
return res;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeComparator.java
index 27487ed..f98f6e0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypeComparator.java
@@ -289,15 +289,16 @@ public abstract class TypeComparator<T> implements Serializable {
/**
* Extracts the key fields from a record. This is for use by the PairComparator to provide
* interoperability between different record types.
+ * @return the number of keys added to target.
*/
- public abstract Object[] extractKeys(T record);
+ public abstract int extractKeys(Object record, Object[] target, int index);
/**
* Get the field comparators. This is used together with {@link #extractKeys(Object)} to provide
* interoperability between different record types.
*/
@SuppressWarnings("rawtypes")
- public abstract TypeComparator[] getComparators();
+ public abstract TypeComparator[] getFlatComparators();
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypePairComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypePairComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypePairComparator.java
index a11da73..fe278fd 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypePairComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/TypePairComparator.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.common.typeutils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BasicTypeComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BasicTypeComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BasicTypeComparator.java
index 4e44326..d1d8e0f 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BasicTypeComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/base/BasicTypeComparator.java
@@ -33,10 +33,6 @@ public abstract class BasicTypeComparator<T extends Comparable<T>> extends TypeC
protected final boolean ascendingComparison;
- // This is used in extractKeys, so that we don't create a new array for every call.
- @SuppressWarnings("rawtypes")
- private final Comparable[] extractedKey = new Comparable[1];
-
// For use by getComparators
@SuppressWarnings("rawtypes")
private final TypeComparator[] comparators = new TypeComparator[] {this};
@@ -89,14 +85,14 @@ public abstract class BasicTypeComparator<T extends Comparable<T>> extends TypeC
}
@Override
- public Object[] extractKeys(T record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
- @Override
@SuppressWarnings("rawtypes")
- public TypeComparator[] getComparators() {
+ @Override
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/record/RecordComparator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/record/RecordComparator.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/record/RecordComparator.java
index 4e1f64a..605d6a1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/record/RecordComparator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/record/RecordComparator.java
@@ -391,13 +391,14 @@ public final class RecordComparator extends TypeComparator<Record> {
}
@Override
- public Object[] extractKeys(Record record) {
+ public int extractKeys(Object record, Object[] target, int index) {
throw new UnsupportedOperationException("Record does not support extactKeys and " +
"getComparators. This cannot be used with the GenericPairComparator.");
}
+
@Override
- public TypeComparator<?>[] getComparators() {
+ public TypeComparator<?>[] getFlatComparators() {
throw new UnsupportedOperationException("Record does not support extactKeys and " +
"getComparators. This cannot be used with the GenericPairComparator.");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-core/src/main/java/org/apache/flink/types/KeyFieldOutOfBoundsException.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/types/KeyFieldOutOfBoundsException.java b/flink-core/src/main/java/org/apache/flink/types/KeyFieldOutOfBoundsException.java
index 1b9f108..3d04efa 100644
--- a/flink-core/src/main/java/org/apache/flink/types/KeyFieldOutOfBoundsException.java
+++ b/flink-core/src/main/java/org/apache/flink/types/KeyFieldOutOfBoundsException.java
@@ -62,6 +62,11 @@ public class KeyFieldOutOfBoundsException extends RuntimeException
this.fieldNumber = fieldNumber;
}
+ public KeyFieldOutOfBoundsException(int fieldNumber, Throwable parent) {
+ super("Field " + fieldNumber + " is accessed for a key, but out of bounds in the record.", parent);
+ this.fieldNumber = fieldNumber;
+ }
+
/**
* Gets the field number that was attempted to access. If the number is not set, this method returns
* {@code -1}.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/pom.xml b/flink-examples/flink-java-examples/pom.xml
index 836b0bc..3ebd85e 100644
--- a/flink-examples/flink-java-examples/pom.xml
+++ b/flink-examples/flink-java-examples/pom.xml
@@ -296,6 +296,30 @@ under the License.
</configuration>
</execution>
+ <!-- WordCountPOJO -->
+ <execution>
+ <id>WordCountPOJO</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <classifier>WordCountPOJO</classifier>
+
+ <archive>
+ <manifestEntries>
+ <program-class>org.apache.flink.examples.java.wordcount.PojoExample</program-class>
+ </manifestEntries>
+ </archive>
+
+ <includes>
+ <include>**/java/wordcount/PojoExample.class</include>
+ <include>**/java/wordcount/PojoExample$*.class</include>
+ <include>**/java/wordcount/util/WordCountData.class</include>
+ </includes>
+ </configuration>
+ </execution>
+
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/environments/CollectionExecutionExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/environments/CollectionExecutionExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/environments/CollectionExecutionExample.java
deleted file mode 100644
index bc260d5..0000000
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/environments/CollectionExecutionExample.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.example.java.environments;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.CollectionEnvironment;
-
-/**
- * This example shows how to use Flink's collection execution functionality.
- * Collection-based execution is an extremely lightweight, non-parallel way to
- * execute programs on small data: The programs are s
- *
- * Because this method of execution spawns no background threads, managed memory,
- * coordinator, or parallel worker, it has a minimal execution footprint.
- */
-public class CollectionExecutionExample {
-
- public static void main(String[] args) throws Exception {
-
- CollectionEnvironment env = new CollectionEnvironment();
-
- env.fromElements("A", "B", "C", "D")
- .map(new MapFunction<String, String>() {
- public String map(String value) {
- return value + " " + 1;
- };
- })
- .print();
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/example/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/example/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/example/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
deleted file mode 100644
index 30c8fc4..0000000
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/example/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.example.java.remotecollectoroutputformat;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.io.RemoteCollectorConsumer;
-import org.apache.flink.api.java.io.RemoteCollectorImpl;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-
-/**
- * Implements the "WordCount" program that computes a simple word occurrence
- * histogram over some sample data and collects the results with an
- * implementation of a {@link RemoteCollectorConsumer}.
- */
-@SuppressWarnings("serial")
-public class RemoteCollectorOutputFormatExample {
-
- public static void main(String[] args) throws Exception {
-
- /**
- * We create a remote {@link ExecutionEnvironment} here, because this
- * OutputFormat is designed for use in a distributed setting. For local
- * use you should consider using the {@link LocalCollectionOutputFormat
- * <T>}.
- */
- final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("<remote>", 6124,
- "/path/to/your/file.jar");
-
- // get input data
- DataSet<String> text = env.fromElements(
- "To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer",
- "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,");
-
- DataSet<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new LineSplitter())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).aggregate(Aggregations.SUM, 1);
-
- // emit result
- RemoteCollectorImpl.collectLocal(counts,
- new RemoteCollectorConsumer<Tuple2<String, Integer>>() {
- // user defined IRemoteCollectorConsumer
- @Override
- public void collect(Tuple2<String, Integer> element) {
- System.out.println("word/occurrences:" + element);
- }
- });
-
- // local collection to store results in
- Set<Tuple2<String, Integer>> collection = new HashSet<Tuple2<String, Integer>>();
- // collect results from remote in local collection
- RemoteCollectorImpl.collectLocal(counts, collection);
-
- // execute program
- env.execute("WordCount Example with RemoteCollectorOutputFormat");
-
- System.out.println(collection);
-
- RemoteCollectorImpl.shutdownAll();
- }
-
- //
- // User Functions
- //
-
- /**
- * Implements the string tokenizer that splits sentences into words as a
- * user-defined FlatMapFunction. The function takes a line (String) and
- * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
- * Integer>).
- */
- public static final class LineSplitter implements
- FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
new file mode 100644
index 0000000..1ce3e7a
--- /dev/null
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.examples.java.environments;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.CollectionEnvironment;
+
+/**
+ * This example shows how to use Flink's collection execution functionality.
+ * Collection-based execution is an extremely lightweight, non-parallel way to
+ * execute programs on small data: The programs are s
+ *
+ * Because this method of execution spawns no background threads, managed memory,
+ * coordinator, or parallel worker, it has a minimal execution footprint.
+ */
+public class CollectionExecutionExample {
+
+ public static void main(String[] args) throws Exception {
+
+ CollectionEnvironment env = new CollectionEnvironment();
+
+ env.fromElements("A", "B", "C", "D")
+ .map(new MapFunction<String, String>() {
+ public String map(String value) {
+ return value + " " + 1;
+ };
+ })
+ .print();
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
new file mode 100644
index 0000000..36b5c82
--- /dev/null
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.examples.java.remotecollectoroutputformat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.aggregation.Aggregations;
+import org.apache.flink.api.java.io.RemoteCollectorConsumer;
+import org.apache.flink.api.java.io.RemoteCollectorImpl;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+/**
+ * Implements the "WordCount" program that computes a simple word occurrence
+ * histogram over some sample data and collects the results with an
+ * implementation of a {@link RemoteCollectorConsumer}.
+ */
+@SuppressWarnings("serial")
+public class RemoteCollectorOutputFormatExample {
+
+ public static void main(String[] args) throws Exception {
+
+ /**
+ * We create a remote {@link ExecutionEnvironment} here, because this
+ * OutputFormat is designed for use in a distributed setting. For local
+ * use you should consider using the {@link LocalCollectionOutputFormat
+ * <T>}.
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("<remote>", 6124,
+ "/path/to/your/file.jar");
+
+ // get input data
+ DataSet<String> text = env.fromElements(
+ "To be, or not to be,--that is the question:--",
+ "Whether 'tis nobler in the mind to suffer",
+ "The slings and arrows of outrageous fortune",
+ "Or to take arms against a sea of troubles,");
+
+ DataSet<Tuple2<String, Integer>> counts =
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new LineSplitter())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0).aggregate(Aggregations.SUM, 1);
+
+ // emit result
+ RemoteCollectorImpl.collectLocal(counts,
+ new RemoteCollectorConsumer<Tuple2<String, Integer>>() {
+ // user defined IRemoteCollectorConsumer
+ @Override
+ public void collect(Tuple2<String, Integer> element) {
+ System.out.println("word/occurrences:" + element);
+ }
+ });
+
+ // local collection to store results in
+ Set<Tuple2<String, Integer>> collection = new HashSet<Tuple2<String, Integer>>();
+ // collect results from remote in local collection
+ RemoteCollectorImpl.collectLocal(counts, collection);
+
+ // execute program
+ env.execute("WordCount Example with RemoteCollectorOutputFormat");
+
+ System.out.println(collection);
+
+ RemoteCollectorImpl.shutdownAll();
+ }
+
+ //
+ // User Functions
+ //
+
+ /**
+ * Implements the string tokenizer that splits sentences into words as a
+ * user-defined FlatMapFunction. The function takes a line (String) and
+ * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
+ * Integer>).
+ */
+ public static final class LineSplitter implements
+ FlatMapFunction<String, Tuple2<String, Integer>> {
+
+ @Override
+ public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<String, Integer>(token, 1));
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
new file mode 100644
index 0000000..a79462e
--- /dev/null
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.examples.java.wordcount;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.core.fs.FileSystem.WriteMode;
+import org.apache.flink.examples.java.wordcount.util.WordCountData;
+import org.apache.flink.util.Collector;
+
+
+/**
+ * This example shows an implementation of Wordcount without using the
+ * Tuple2 type, but a custom class.
+ *
+ */
+@SuppressWarnings("serial")
+public class PojoExample {
+
+ /**
+ * This is the POJO (Plain Old Java Object) that is bein used
+ * for all the operations.
+ * As long as all fields are public or have a getter/setter, the system can handle them
+ */
+ public static class Word {
+ // fields
+ private String word;
+ private Integer frequency;
+
+ // constructors
+ public Word() {
+ }
+ public Word(String word, int i) {
+ this.word = word;
+ this.frequency = i;
+ }
+ // getters setters
+ public String getWord() {
+ return word;
+ }
+ public void setWord(String word) {
+ this.word = word;
+ }
+ public Integer getFrequency() {
+ return frequency;
+ }
+ public void setFrequency(Integer frequency) {
+ this.frequency = frequency;
+ }
+ // to String
+ @Override
+ public String toString() {
+ return "Word="+word+" freq="+frequency;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ if(!parseParameters(args)) {
+ return;
+ }
+
+ // set up the execution environment
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ // get input data
+ DataSet<String> text = getTextDataSet(env);
+
+ DataSet<Word> counts =
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new Tokenizer())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy("word")
+ .reduce(new ReduceFunction<Word>() {
+ @Override
+ public Word reduce(Word value1, Word value2) throws Exception {
+ return new Word(value1.word,value1.frequency + value2.frequency);
+ }
+ });
+
+ if(fileOutput) {
+ counts.writeAsText(outputPath, WriteMode.OVERWRITE);
+ } else {
+ counts.print();
+ }
+
+ // execute program
+ env.execute("WordCount-Pojo Example");
+ }
+
+ // *************************************************************************
+ // USER FUNCTIONS
+ // *************************************************************************
+
+ /**
+ * Implements the string tokenizer that splits sentences into words as a user-defined
+ * FlatMapFunction. The function takes a line (String) and splits it into
+ * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
+ */
+ public static final class Tokenizer implements FlatMapFunction<String, Word> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Word> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Word(token, 1));
+ }
+ }
+ }
+ }
+
+ // *************************************************************************
+ // UTIL METHODS
+ // *************************************************************************
+
+ private static boolean fileOutput = false;
+ private static String textPath;
+ private static String outputPath;
+
+ private static boolean parseParameters(String[] args) {
+
+ if(args.length > 0) {
+ // parse input arguments
+ fileOutput = true;
+ if(args.length == 2) {
+ textPath = args[0];
+ outputPath = args[1];
+ } else {
+ System.err.println("Usage: WordCount <text path> <result path>");
+ return false;
+ }
+ } else {
+ System.out.println("Executing WordCount example with built-in default data.");
+ System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println(" Usage: WordCount <text path> <result path>");
+ }
+ return true;
+ }
+
+ private static DataSet<String> getTextDataSet(ExecutionEnvironment env) {
+ if(fileOutput) {
+ // read the text file from given input path
+ return env.readTextFile(textPath);
+ } else {
+ // get default test text data
+ return WordCountData.getDefaultTextLineDataSet(env);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index d2939d8..424d30b 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -430,7 +430,7 @@ public abstract class DataSet<T> {
* @return A DistinctOperator that represents the distinct DataSet.
*/
public DistinctOperator<T> distinct(int... fields) {
- return new DistinctOperator<T>(this, new Keys.FieldPositionKeys<T>(fields, getType(), true));
+ return new DistinctOperator<T>(this, new Keys.ExpressionKeys<T>(fields, getType(), true));
}
/**
@@ -500,7 +500,7 @@ public abstract class DataSet<T> {
* @see DataSet
*/
public UnsortedGrouping<T> groupBy(int... fields) {
- return new UnsortedGrouping<T>(this, new Keys.FieldPositionKeys<T>(fields, getType(), false));
+ return new UnsortedGrouping<T>(this, new Keys.ExpressionKeys<T>(fields, getType(), false));
}
/**
@@ -526,9 +526,9 @@ public abstract class DataSet<T> {
* @see org.apache.flink.api.java.operators.GroupReduceOperator
* @see DataSet
*/
-// public UnsortedGrouping<T> groupBy(String... fields) {
-// return new UnsortedGrouping<T>(this, new Keys.ExpressionKeys<T>(fields, getType()));
-// }
+ public UnsortedGrouping<T> groupBy(String... fields) {
+ return new UnsortedGrouping<T>(this, new Keys.ExpressionKeys<T>(fields, getType()));
+ }
// --------------------------------------------------------------------------------------------
// Joining
@@ -541,7 +541,7 @@ public abstract class DataSet<T> {
* joining elements into one DataSet.</br>
*
* This method returns a {@link JoinOperatorSets} on which
- * {@link JoinOperatorSets#where(int...)} needs to be called to define the join key of the first
+ * {@link JoinOperatorSets#where()} needs to be called to define the join key of the first
* joining (i.e., this) DataSet.
*
* @param other The other DataSet with which this DataSet is joined.
@@ -562,7 +562,7 @@ public abstract class DataSet<T> {
* This method also gives the hint to the optimizer that the second DataSet to join is much
* smaller than the first one.</br>
* This method returns a {@link JoinOperatorSets} on which
- * {@link JoinOperatorSets#where(int...)} needs to be called to define the join key of the first
+ * {@link JoinOperatorSets#where()} needs to be called to define the join key of the first
* joining (i.e., this) DataSet.
*
* @param other The other DataSet with which this DataSet is joined.
@@ -583,7 +583,7 @@ public abstract class DataSet<T> {
* This method also gives the hint to the optimizer that the second DataSet to join is much
* larger than the first one.</br>
* This method returns a {@link JoinOperatorSets JoinOperatorSet} on which
- * {@link JoinOperatorSets#where(int...)} needs to be called to define the join key of the first
+ * {@link JoinOperatorSets#where()} needs to be called to define the join key of the first
* joining (i.e., this) DataSet.
*
* @param other The other DataSet with which this DataSet is joined.
@@ -610,7 +610,7 @@ public abstract class DataSet<T> {
* The CoGroupFunction can iterate over the elements of both groups and return any number
* of elements including none.</br>
* This method returns a {@link CoGroupOperatorSets} on which
- * {@link CoGroupOperatorSets#where(int...)} needs to be called to define the grouping key of the first
+ * {@link CoGroupOperatorSets#where()} needs to be called to define the grouping key of the first
* (i.e., this) DataSet.
*
* @param other The other DataSet of the CoGroup transformation.
@@ -814,7 +814,7 @@ public abstract class DataSet<T> {
* @see org.apache.flink.api.java.operators.DeltaIteration
*/
public <R> DeltaIteration<T, R> iterateDelta(DataSet<R> workset, int maxIterations, int... keyPositions) {
- Keys.FieldPositionKeys<T> keys = new Keys.FieldPositionKeys<T>(keyPositions, getType(), false);
+ Keys.ExpressionKeys<T> keys = new Keys.ExpressionKeys<T>(keyPositions, getType(), false);
return new DeltaIteration<T, R>(getExecutionEnvironment(), getType(), this, workset, keys, maxIterations);
}
@@ -863,7 +863,7 @@ public abstract class DataSet<T> {
* @return The partitioned DataSet.
*/
public PartitionOperator<T> partitionByHash(int... fields) {
- return new PartitionOperator<T>(this, PartitionMethod.HASH, new Keys.FieldPositionKeys<T>(fields, getType(), false));
+ return new PartitionOperator<T>(this, PartitionMethod.HASH, new Keys.ExpressionKeys<T>(fields, getType(), false));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
index c1d0728..54e36c0 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/ExecutionEnvironment.java
@@ -791,4 +791,5 @@ public abstract class ExecutionEnvironment {
public static boolean localExecutionIsAllowed() {
return allowLocalExecution;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java b/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
new file mode 100644
index 0000000..a81f55d
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Field;
+
+public class SerializationSpeedTest {
+
+ static Field wordDescField;
+ static Field wordField;
+ static {
+ try {
+ wordDescField = WC.class.getField("wordDesc");
+ wordField = ComplexWordDescriptor.class.getField("word");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static class ComplexWordDescriptor {
+ public String word;
+
+ public String getWord() {
+ return word;
+ }
+ }
+
+ public static class WC {
+ public int count;
+ public ComplexWordDescriptor wordDesc;
+
+ public WC(int c, String s) throws NoSuchFieldException,
+ SecurityException {
+ this.count = c;
+ this.wordDesc = new ComplexWordDescriptor();
+ this.wordDesc.word = s;
+ }
+
+ public ComplexWordDescriptor getWordDesc() {
+ return wordDesc;
+ }
+
+ }
+
+ public static int compareCodeGenPublicFields(WC w1, WC w2) {
+ return w1.wordDesc.word.compareTo(w2.wordDesc.word);
+ }
+
+ public static int compareCodeGenMethods(WC w1, WC w2) {
+ return w1.getWordDesc().getWord().compareTo(w2.getWordDesc().getWord());
+ }
+
+ public static int compareReflective(WC w1, WC w2)
+ throws IllegalArgumentException, IllegalAccessException {
+ // get String of w1
+ Object wordDesc1 = wordDescField.get(w1);
+ String word2cmp1 = (String) wordField.get(wordDesc1);
+
+ // get String of w2
+ Object wordDesc2 = wordDescField.get(w2);
+ String word2cmp2 = (String) wordField.get(wordDesc2);
+
+ return word2cmp1.compareTo(word2cmp2);
+ }
+
+ /**
+ * results on Core i7 2600k
+ *
+ *
+ * warming up Code gen 5019 Reflection 20364 Factor = 4.057382
+ */
+ public static void main(String[] args) throws NoSuchFieldException,
+ SecurityException, IllegalArgumentException, IllegalAccessException {
+ final long RUNS = 1000000000L;
+
+ final RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
+ String jvm = bean.getVmName() + " - " + bean.getVmVendor() + " - "
+ + bean.getSpecVersion() + '/' + bean.getVmVersion();
+ System.err.println("Jvm info : " + jvm);
+
+ WC word0 = new WC(14, "Hallo");
+ WC word1 = new WC(3, "Hola");
+
+ System.err.println("warming up");
+ for (long i = 0; i < 100000000; i++) {
+ compareCodeGenPublicFields(word0, word1);
+ compareCodeGenMethods(word0, word1);
+ compareReflective(word0, word1);
+ }
+
+ System.err.println("Code gen public fields");
+ long startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareCodeGenPublicFields(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ long stopTime = System.currentTimeMillis();
+ long elapsedTimeGen = stopTime - startTime;
+ System.err.println(elapsedTimeGen);
+
+ System.err.println("Code gen methods");
+ startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareCodeGenPublicFields(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ stopTime = System.currentTimeMillis();
+ long elapsedTimeGenMethods = stopTime - startTime;
+ System.err.println(elapsedTimeGenMethods);
+
+ System.err.println("Reflection");
+
+ startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareReflective(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ stopTime = System.currentTimeMillis();
+ long elapsedTimeRef = stopTime - startTime;
+ System.err.println(elapsedTimeRef);
+
+ System.err.println("Factor vs public = "
+ + (elapsedTimeRef / (float) elapsedTimeGen));
+ System.err.println("Factor vs methods = "
+ + (elapsedTimeRef / (float) elapsedTimeGenMethods));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
index ee9b65f..041dc75 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java
@@ -195,7 +195,7 @@ public class AggregateOperator<IN> extends SingleInputOperator<IN, IN, Aggregate
return po;
}
- if (this.grouping.getKeys() instanceof Keys.FieldPositionKeys) {
+ if (this.grouping.getKeys() instanceof Keys.ExpressionKeys) {
// grouped aggregation
int[] logicalKeyPositions = this.grouping.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getResultType());
[06/11] [FLINK-1032] Rework support for POJO types in the Java API
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
index 9ae2d6b..9d7eed4 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoComparator.java
@@ -22,7 +22,9 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.lang.reflect.Field;
+import java.util.List;
+import org.apache.flink.api.common.typeutils.CompositeTypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.core.memory.DataInputView;
@@ -32,10 +34,11 @@ import org.apache.flink.types.NullKeyFieldException;
import org.apache.flink.util.InstantiationUtil;
-public final class PojoComparator<T> extends TypeComparator<T> implements java.io.Serializable {
-
+public final class PojoComparator<T> extends CompositeTypeComparator<T> implements java.io.Serializable {
+
private static final long serialVersionUID = 1L;
+ // Reflection fields for the comp fields
private transient Field[] keyFields;
private final TypeComparator<Object>[] comparators;
@@ -52,8 +55,6 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
private final Class<T> type;
- private final Comparable[] extractedKeys;
-
@SuppressWarnings("unchecked")
public PojoComparator(Field[] keyFields, TypeComparator<?>[] comparators, TypeSerializer<T> serializer, Class<T> type) {
this.keyFields = keyFields;
@@ -70,6 +71,12 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
for (int i = 0; i < this.comparators.length; i++) {
TypeComparator<?> k = this.comparators[i];
+ if(k == null) {
+ throw new IllegalArgumentException("One of the passed comparators is null");
+ }
+ if(keyFields[i] == null) {
+ throw new IllegalArgumentException("One of the passed reflection fields is null");
+ }
// as long as the leading keys support normalized keys, we can build up the composite key
if (k.supportsNormalizedKey()) {
@@ -102,8 +109,6 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
this.numLeadingNormalizableKeys = nKeys;
this.normalizableKeyPrefixLen = nKeyLen;
this.invertNormKey = inverted;
-
- extractedKeys = new Comparable[keyFields.length];
}
@SuppressWarnings("unchecked")
@@ -130,8 +135,6 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
} catch (ClassNotFoundException e) {
throw new RuntimeException("Cannot copy serializer", e);
}
-
- extractedKeys = new Comparable[keyFields.length];
}
private void writeObject(ObjectOutputStream out)
@@ -150,87 +153,87 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
int numKeyFields = in.readInt();
keyFields = new Field[numKeyFields];
for (int i = 0; i < numKeyFields; i++) {
- Class<?> clazz = (Class<?>)in.readObject();
+ Class<?> clazz = (Class<?>) in.readObject();
String fieldName = in.readUTF();
- keyFields[i] = null;
// try superclasses as well
while (clazz != null) {
try {
- keyFields[i] = clazz.getDeclaredField(fieldName);
- keyFields[i].setAccessible(true);
+ Field field = clazz.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ keyFields[i] = field;
break;
} catch (NoSuchFieldException e) {
clazz = clazz.getSuperclass();
}
}
- if (keyFields[i] == null) {
+ if (keyFields[i] == null ) {
throw new RuntimeException("Class resolved at TaskManager is not compatible with class read during Plan setup."
+ " (" + fieldName + ")");
}
}
}
-
public Field[] getKeyFields() {
return this.keyFields;
}
- public TypeComparator[] getComparators() {
- return this.comparators;
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Override
+ public void getFlatComparator(List<TypeComparator> flatComparators) {
+ for(int i = 0; i < comparators.length; i++) {
+ if(comparators[i] instanceof CompositeTypeComparator) {
+ ((CompositeTypeComparator)comparators[i]).getFlatComparator(flatComparators);
+ } else {
+ flatComparators.add(comparators[i]);
+ }
+ }
+ }
+
+ /**
+ * This method is handling the IllegalAccess exceptions of Field.get()
+ */
+ private final Object accessField(Field field, Object object) {
+ try {
+ object = field.get(object);
+ } catch (NullPointerException npex) {
+ throw new NullKeyFieldException("Unable to access field "+field+" on object "+object);
+ } catch (IllegalAccessException iaex) {
+ throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo."
+ + " fiels: " + field + " obj: " + object);
+ }
+ return object;
}
@Override
public int hash(T value) {
int i = 0;
- try {
- int code = 0;
- for (; i < this.keyFields.length; i++) {
- code ^= this.comparators[i].hash(this.keyFields[i].get(value));
- code *= HASH_SALT[i & 0x1F]; // salt code with (i % HASH_SALT.length)-th salt component
- }
- return code;
- }
- catch (NullPointerException npex) {
- throw new NullKeyFieldException(this.keyFields[i].toString());
- }
- catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
+ int code = 0;
+ for (; i < this.keyFields.length; i++) {
+ code *= TupleComparatorBase.HASH_SALT[i & 0x1F];
+ code += this.comparators[i].hash(accessField(keyFields[i], value));
+
}
+ return code;
+
}
@Override
public void setReference(T toCompare) {
int i = 0;
- try {
- for (; i < this.keyFields.length; i++) {
- this.comparators[i].setReference(this.keyFields[i].get(toCompare));
- }
- }
- catch (NullPointerException npex) {
- throw new NullKeyFieldException(this.keyFields[i].toString());
- }
- catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
+ for (; i < this.keyFields.length; i++) {
+ this.comparators[i].setReference(accessField(keyFields[i], toCompare));
}
}
@Override
public boolean equalToReference(T candidate) {
int i = 0;
- try {
- for (; i < this.keyFields.length; i++) {
- if (!this.comparators[i].equalToReference(this.keyFields[i].get(candidate))) {
- return false;
- }
+ for (; i < this.keyFields.length; i++) {
+ if (!this.comparators[i].equalToReference(accessField(keyFields[i], candidate))) {
+ return false;
}
- return true;
- }
- catch (NullPointerException npex) {
- throw new NullKeyFieldException(this.keyFields[i].toString());
- }
- catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
}
+ return true;
}
@Override
@@ -255,22 +258,17 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
@Override
public int compare(T first, T second) {
int i = 0;
- try {
- for (; i < keyFields.length; i++) {
- int cmp = comparators[i].compare(keyFields[i].get(first),keyFields[i].get(second));
- if (cmp != 0) {
- return cmp;
- }
+ for (; i < keyFields.length; i++) {
+ int cmp = comparators[i].compare(accessField(keyFields[i], first), accessField(keyFields[i], second));
+ if (cmp != 0) {
+ return cmp;
}
-
- return 0;
- } catch (NullPointerException npex) {
- throw new NullKeyFieldException(keyFields[i].toString() + " " + first.toString() + " " + second.toString());
- } catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
}
+
+ return 0;
}
+
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
T first = this.serializer.createInstance();
@@ -302,21 +300,13 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
@Override
public void putNormalizedKey(T value, MemorySegment target, int offset, int numBytes) {
int i = 0;
- try {
- for (; i < this.numLeadingNormalizableKeys & numBytes > 0; i++)
- {
- int len = this.normalizedKeyLengths[i];
- len = numBytes >= len ? len : numBytes;
- this.comparators[i].putNormalizedKey(this.keyFields[i].get(value), target, offset, len);
- numBytes -= len;
- offset += len;
- }
- }
- catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
- }
- catch (NullPointerException npex) {
- throw new NullKeyFieldException(this.keyFields[i].toString());
+ for (; i < this.numLeadingNormalizableKeys & numBytes > 0; i++)
+ {
+ int len = this.normalizedKeyLengths[i];
+ len = numBytes >= len ? len : numBytes;
+ this.comparators[i].putNormalizedKey(accessField(keyFields[i], value), target, offset, len);
+ numBytes -= len;
+ offset += len;
}
}
@@ -347,36 +337,21 @@ public final class PojoComparator<T> extends TypeComparator<T> implements java.i
}
@Override
- public Object[] extractKeys(T record) {
- int i = 0;
- try {
- for (; i < keyFields.length; i++) {
- extractedKeys[i] = (Comparable) keyFields[i].get(record);
+ public int extractKeys(Object record, Object[] target, int index) {
+ int localIndex = index;
+ for (int i = 0; i < comparators.length; i++) {
+ if(comparators[i] instanceof PojoComparator || comparators[i] instanceof TupleComparator) {
+ localIndex += comparators[i].extractKeys(accessField(keyFields[i], record), target, localIndex) -1;
+ } else {
+ // non-composite case (= atomic). We can assume this to have only one key.
+ // comparators[i].extractKeys(accessField(keyFields[i], record), target, i);
+ target[localIndex] = accessField(keyFields[i], record);
}
+ localIndex++;
}
- catch (IllegalAccessException iaex) {
- throw new RuntimeException("This should not happen since we call setAccesssible(true) in PojoTypeInfo.");
- }
- catch (NullPointerException npex) {
- throw new NullKeyFieldException(this.keyFields[i].toString());
- }
- return extractedKeys;
+ return localIndex - index;
}
// --------------------------------------------------------------------------------------------
-
- /**
- * A sequence of prime numbers to be used for salting the computed hash values.
- * Based on some empirical evidence, we are using a 32-element subsequence of the
- * OEIS sequence #A068652 (numbers such that every cyclic permutation is a prime).
- *
- * @see: http://en.wikipedia.org/wiki/List_of_prime_numbers
- * @see: http://oeis.org/A068652
- */
- private static final int[] HASH_SALT = new int[] {
- 73 , 79 , 97 , 113 , 131 , 197 , 199 , 311 ,
- 337 , 373 , 719 , 733 , 919 , 971 , 991 , 1193 ,
- 1931 , 3119 , 3779 , 7793 , 7937 , 9311 , 9377 , 11939 ,
- 19391, 19937, 37199, 39119, 71993, 91193, 93719, 93911 };
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index 71f2cd8..99b9f65 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -174,11 +174,22 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
@Override
public void serialize(T value, DataOutputView target) throws IOException {
+ // handle null values
+ if (value == null) {
+ target.writeBoolean(true);
+ return;
+ } else {
+ target.writeBoolean(false);
+ }
try {
-
for (int i = 0; i < numFields; i++) {
Object o = fields[i].get(value);
- fieldSerializers[i].serialize(o, target);
+ if(o == null) {
+ target.writeBoolean(true); // null field handling
+ } else {
+ target.writeBoolean(false);
+ fieldSerializers[i].serialize(o, target);
+ }
}
} catch (IllegalAccessException e) {
throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" +
@@ -188,6 +199,10 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
@Override
public T deserialize(DataInputView source) throws IOException {
+ boolean isNull = source.readBoolean();
+ if(isNull) {
+ return null;
+ }
T target;
try {
target = clazz.newInstance();
@@ -198,8 +213,13 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
try {
for (int i = 0; i < numFields; i++) {
- Object field = fieldSerializers[i].deserialize(source);
- fields[i].set(target, field);
+ isNull = source.readBoolean();
+ if(isNull) {
+ fields[i].set(target, null);
+ } else {
+ Object field = fieldSerializers[i].deserialize(source);
+ fields[i].set(target, field);
+ }
}
} catch (IllegalAccessException e) {
throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" +
@@ -210,10 +230,20 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
@Override
public T deserialize(T reuse, DataInputView source) throws IOException {
+ // handle null values
+ boolean isNull = source.readBoolean();
+ if (isNull) {
+ return null;
+ }
try {
for (int i = 0; i < numFields; i++) {
- Object field = fieldSerializers[i].deserialize(fields[i].get(reuse), source);
- fields[i].set(reuse, field);
+ isNull = source.readBoolean();
+ if(isNull) {
+ fields[i].set(reuse, null);
+ } else {
+ Object field = fieldSerializers[i].deserialize(fields[i].get(reuse), source);
+ fields[i].set(reuse, field);
+ }
}
} catch (IllegalAccessException e) {
throw new RuntimeException("Error during POJO copy, this should not happen since we check the fields" +
@@ -224,7 +254,10 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
+ // copy the Non-Null/Null tag
+ target.writeBoolean(source.readBoolean());
for (int i = 0; i < numFields; i++) {
+ target.writeBoolean(source.readBoolean());
fieldSerializers[i].copy(source, target);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/RuntimePairComparatorFactory.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/RuntimePairComparatorFactory.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/RuntimePairComparatorFactory.java
index eee6643..31e28f7 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/RuntimePairComparatorFactory.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/RuntimePairComparatorFactory.java
@@ -28,7 +28,6 @@ public final class RuntimePairComparatorFactory<T1, T2>
private static final long serialVersionUID = 1L;
- @SuppressWarnings("unchecked")
@Override
public TypePairComparator<T1, T2> createComparator12(
TypeComparator<T1> comparator1,
@@ -36,7 +35,6 @@ public final class RuntimePairComparatorFactory<T1, T2>
return new GenericPairComparator<T1, T2>(comparator1, comparator2);
}
- @SuppressWarnings("unchecked")
@Override
public TypePairComparator<T2, T1> createComparator21(
TypeComparator<T1> comparator1,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
index f9b4084..61a1567 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparator.java
@@ -30,26 +30,20 @@ import org.apache.flink.types.NullKeyFieldException;
public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<T> {
private static final long serialVersionUID = 1L;
-
- private final Object[] extractedKeys;
-
- @SuppressWarnings("unchecked")
+
public TupleComparator(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?>[] serializers) {
super(keyPositions, comparators, serializers);
- extractedKeys = new Object[keyPositions.length];
}
- @SuppressWarnings("unchecked")
private TupleComparator(TupleComparator<T> toClone) {
super(toClone);
- extractedKeys = new Object[keyPositions.length];
-
}
// --------------------------------------------------------------------------------------------
// Comparator Methods
// --------------------------------------------------------------------------------------------
+ @SuppressWarnings("unchecked")
@Override
public int hash(T value) {
int i = 0;
@@ -70,6 +64,7 @@ public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<
}
}
+ @SuppressWarnings("unchecked")
@Override
public void setReference(T toCompare) {
int i = 0;
@@ -86,6 +81,7 @@ public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<
}
}
+ @SuppressWarnings("unchecked")
@Override
public boolean equalToReference(T candidate) {
int i = 0;
@@ -105,6 +101,7 @@ public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<
}
}
+ @SuppressWarnings("unchecked")
@Override
public int compare(T first, T second) {
int i = 0;
@@ -127,6 +124,7 @@ public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<
}
}
+ @SuppressWarnings("unchecked")
@Override
public void putNormalizedKey(T value, MemorySegment target, int offset, int numBytes) {
int i = 0;
@@ -146,11 +144,19 @@ public final class TupleComparator<T extends Tuple> extends TupleComparatorBase<
}
@Override
- public Object[] extractKeys(T record) {
- for (int i = 0; i < keyPositions.length; i++) {
- extractedKeys[i] = record.getField(keyPositions[i]);
+ public int extractKeys(Object record, Object[] target, int index) {
+ int localIndex = index;
+ for(int i = 0; i < comparators.length; i++) {
+ // handle nested case
+ if(comparators[i] instanceof TupleComparator || comparators[i] instanceof PojoComparator) {
+ localIndex += comparators[i].extractKeys(((Tuple) record).getField(keyPositions[i]), target, localIndex) -1;
+ } else {
+ // flat
+ target[localIndex] = ((Tuple) record).getField(keyPositions[i]);
+ }
+ localIndex++;
}
- return extractedKeys;
+ return localIndex - index;
}
public TypeComparator<T> duplicate() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparatorBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparatorBase.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparatorBase.java
index 4c26318..abcf89c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparatorBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/TupleComparatorBase.java
@@ -15,9 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.java.typeutils.runtime;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.flink.api.common.typeutils.CompositeTypeComparator;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
@@ -26,15 +29,16 @@ import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.types.KeyFieldOutOfBoundsException;
import org.apache.flink.types.NullKeyFieldException;
-import java.io.IOException;
+public abstract class TupleComparatorBase<T> extends CompositeTypeComparator<T> implements java.io.Serializable {
-public abstract class TupleComparatorBase<T> extends TypeComparator<T> implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
/** key positions describe which fields are keys in what order */
protected int[] keyPositions;
/** comparators for the key fields, in the same order as the key fields */
+ @SuppressWarnings("rawtypes")
protected TypeComparator[] comparators;
/** serializer factories to duplicate non thread-safe serializers */
@@ -51,6 +55,7 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
/** serializers to deserialize the first n fields for comparison */
+ @SuppressWarnings("rawtypes")
protected transient TypeSerializer[] serializers;
// cache for the deserialized field objects
@@ -115,7 +120,6 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
this.invertNormKey = inverted;
}
- @SuppressWarnings("unchecked")
protected TupleComparatorBase(TupleComparatorBase<T> toClone) {
privateDuplicate(toClone);
}
@@ -146,14 +150,23 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
return this.keyPositions;
}
- public TypeComparator[] getComparators() {
- return this.comparators;
- }
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Override
+ public void getFlatComparator(List<TypeComparator> flatComparators) {
+ for(int i = 0; i < comparators.length; i++) {
+ if(comparators[i] instanceof CompositeTypeComparator) {
+ ((CompositeTypeComparator)comparators[i]).getFlatComparator(flatComparators);
+ } else {
+ flatComparators.add(comparators[i]);
+ }
+ }
+ }
// --------------------------------------------------------------------------------------------
// Comparator Methods
// --------------------------------------------------------------------------------------------
+
@Override
public int compareToReference(TypeComparator<T> referencedComparator) {
TupleComparatorBase<T> other = (TupleComparatorBase<T>) referencedComparator;
@@ -161,6 +174,7 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
int i = 0;
try {
for (; i < this.keyPositions.length; i++) {
+ @SuppressWarnings("unchecked")
int cmp = this.comparators[i].compareToReference(other.comparators[i]);
if (cmp != 0) {
return cmp;
@@ -176,6 +190,7 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
}
}
+ @SuppressWarnings("unchecked")
@Override
public int compareSerialized(DataInputView firstSource, DataInputView secondSource) throws IOException {
if (deserializedFields1 == null) {
@@ -201,7 +216,7 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
} catch (NullPointerException npex) {
throw new NullKeyFieldException(keyPositions[i]);
} catch (IndexOutOfBoundsException iobex) {
- throw new KeyFieldOutOfBoundsException(keyPositions[i]);
+ throw new KeyFieldOutOfBoundsException(keyPositions[i], iobex);
}
}
@@ -245,7 +260,6 @@ public abstract class TupleComparatorBase<T> extends TypeComparator<T> implement
// --------------------------------------------------------------------------------------------
- @SuppressWarnings("unchecked")
protected final void instantiateDeserializationUtils() {
if (this.serializers == null) {
this.serializers = new TypeSerializer[this.serializerFactories.length];
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueComparator.java
index a510863..eca1e6c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/ValueComparator.java
@@ -47,8 +47,7 @@ public class ValueComparator<T extends Value & Comparable<T>> extends TypeCompar
private transient Kryo kryo;
- private final Comparable[] extractedKey = new Comparable[1];
-
+ @SuppressWarnings("rawtypes")
private final TypeComparator[] comparators = new TypeComparator[] {this};
public ValueComparator(boolean ascending, Class<T> type) {
@@ -145,13 +144,14 @@ public class ValueComparator<T extends Value & Comparable<T>> extends TypeCompar
}
@Override
- public Object[] extractKeys(T record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
+ @SuppressWarnings("rawtypes")
@Override
- public TypeComparator[] getComparators() {
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableComparator.java
index a8c4ef5..88985bb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/WritableComparator.java
@@ -44,8 +44,7 @@ public class WritableComparator<T extends Writable & Comparable<T>> extends Type
private transient Kryo kryo;
- private final Comparable[] extractedKey = new Comparable[1];
-
+ @SuppressWarnings("rawtypes")
private final TypeComparator[] comparators = new TypeComparator[] {this};
public WritableComparator(boolean ascending, Class<T> type) {
@@ -129,12 +128,13 @@ public class WritableComparator<T extends Writable & Comparable<T>> extends Type
}
@Override
- public Object[] extractKeys(T record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
- @Override public TypeComparator[] getComparators() {
+ @SuppressWarnings("rawtypes")
+ @Override public TypeComparator[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
index 898eae5..d686633 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/CoGroupOperatorTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.api.java.operator;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@@ -29,10 +28,10 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operator.JoinOperatorTest.CustomType;
@SuppressWarnings("serial")
public class CoGroupOperatorTest {
@@ -127,7 +126,6 @@ public class CoGroupOperatorTest {
ds1.coGroup(ds2).where(5).equalTo(0);
}
- @Ignore
@Test
public void testCoGroupKeyExpressions1() {
@@ -137,13 +135,12 @@ public class CoGroupOperatorTest {
// should work
try {
-// ds1.coGroup(ds2).where("myInt").equalTo("myInt");
+ ds1.coGroup(ds2).where("myInt").equalTo("myInt");
} catch(Exception e) {
Assert.fail();
}
}
- @Ignore
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyExpressions2() {
@@ -152,10 +149,9 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible cogroup key types
-// ds1.coGroup(ds2).where("myInt").equalTo("myString");
+ ds1.coGroup(ds2).where("myInt").equalTo("myString");
}
- @Ignore
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyExpressions3() {
@@ -164,10 +160,9 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible number of cogroup keys
-// ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
+ ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString");
}
- @Ignore
@Test(expected = IllegalArgumentException.class)
public void testCoGroupKeyExpressions4() {
@@ -176,9 +171,58 @@ public class CoGroupOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, cogroup key non-existent
-// ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
+ ds1.coGroup(ds2).where("myNonExistent").equalTo("myInt");
+ }
+
+ @Test
+ public void testCoGroupKeyExpressions1Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should work
+ try {
+ ds1.coGroup(ds2).where("nested.myInt").equalTo("nested.myInt");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
}
+ @Test(expected = InvalidProgramException.class)
+ public void testCoGroupKeyExpressions2Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, incompatible cogroup key types
+ ds1.coGroup(ds2).where("nested.myInt").equalTo("nested.myString");
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testCoGroupKeyExpressions3Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, incompatible number of cogroup keys
+ ds1.coGroup(ds2).where("nested.myInt", "nested.myString").equalTo("nested.myString");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCoGroupKeyExpressions4Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, cogroup key non-existent
+ ds1.coGroup(ds2).where("nested.myNonExistent").equalTo("nested.myInt");
+ }
+
@Test
public void testCoGroupKeySelectors1() {
@@ -304,26 +348,4 @@ public class CoGroupOperatorTest {
}
);
}
-
- public static class CustomType implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public int myInt;
- public long myLong;
- public String myString;
-
- public CustomType() {};
-
- public CustomType(int i, long l, String s) {
- myInt = i;
- myLong = l;
- myString = s;
- }
-
- @Override
- public String toString() {
- return myInt+","+myLong+","+myString;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
index adc8917..c958680 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/GroupingTest.java
@@ -32,7 +32,6 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
public class GroupingTest {
@@ -112,7 +111,6 @@ public class GroupingTest {
tupleDs.groupBy(-1);
}
- @Ignore
@Test
public void testGroupByKeyExpressions1() {
@@ -124,24 +122,22 @@ public class GroupingTest {
// should work
try {
-// ds.groupBy("myInt");
+ ds.groupBy("myInt");
} catch(Exception e) {
Assert.fail();
}
}
- @Ignore
- @Test(expected = UnsupportedOperationException.class)
+ @Test(expected = IllegalArgumentException.class)
public void testGroupByKeyExpressions2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
// should not work: groups on basic type
-// longDs.groupBy("myInt");
+ longDs.groupBy("myInt");
}
- @Ignore
@Test(expected = InvalidProgramException.class)
public void testGroupByKeyExpressions3() {
@@ -150,12 +146,11 @@ public class GroupingTest {
this.customTypeData.add(new CustomType());
DataSet<CustomType> customDs = env.fromCollection(customTypeData);
- // should not work: groups on custom type
+ // should not work: tuple selector on custom type
customDs.groupBy(0);
}
- @Ignore
@Test(expected = IllegalArgumentException.class)
public void testGroupByKeyExpressions4() {
@@ -163,7 +158,34 @@ public class GroupingTest {
DataSet<CustomType> ds = env.fromCollection(customTypeData);
// should not work, key out of tuple bounds
-// ds.groupBy("myNonExistent");
+ ds.groupBy("myNonExistent");
+ }
+
+ @Test
+ public void testGroupByKeyExpressions1Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ this.customTypeData.add(new CustomType());
+
+ DataSet<CustomType> ds = env.fromCollection(customTypeData);
+
+ // should work
+ try {
+ ds.groupBy("nested.myInt");
+ } catch(Exception e) {
+ Assert.fail();
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testGroupByKeyExpressions2Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds = env.fromCollection(customTypeData);
+
+ // should not work, key out of tuple bounds
+ ds.groupBy("nested.myNonExistent");
}
@@ -309,11 +331,15 @@ public class GroupingTest {
public static class CustomType implements Serializable {
+ public static class Nest {
+ public int myInt;
+ }
private static final long serialVersionUID = 1L;
public int myInt;
public long myLong;
public String myString;
+ public Nest nested;
public CustomType() {};
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
index 962755e..de50fd8 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/JoinOperatorTest.java
@@ -22,17 +22,20 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
@SuppressWarnings("serial")
public class JoinOperatorTest {
@@ -41,20 +44,51 @@ public class JoinOperatorTest {
private static final List<Tuple5<Integer, Long, String, Long, Integer>> emptyTupleData =
new ArrayList<Tuple5<Integer, Long, String, Long, Integer>>();
- private final TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>> tupleTypeInfo = new
- TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>>(
+ private final TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>> tupleTypeInfo =
+ new TupleTypeInfo<Tuple5<Integer, Long, String, Long, Integer>>(
BasicTypeInfo.INT_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.STRING_TYPE_INFO,
BasicTypeInfo.LONG_TYPE_INFO,
BasicTypeInfo.INT_TYPE_INFO
);
+ // TUPLE DATA with nested Tuple2
+ private static final List<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>> emptyNestedTupleData =
+ new ArrayList<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>>();
+ private final TupleTypeInfo<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>> nestedTupleTypeInfo =
+ new TupleTypeInfo<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>>(
+ new TupleTypeInfo<Tuple2<Integer, String>> (BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO),
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ // TUPLE DATA with nested CustomType
+ private static final List<Tuple5<CustomType, Long, String, Long, Integer>> emptyNestedCustomTupleData =
+ new ArrayList<Tuple5<CustomType, Long, String, Long, Integer>>();
+
+ private final TupleTypeInfo<Tuple5<CustomType, Long, String, Long, Integer>> nestedCustomTupleTypeInfo =
+ new TupleTypeInfo<Tuple5<CustomType, Long, String, Long, Integer>>(
+ TypeExtractor.getForClass(CustomType.class),
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO
+ );
+
+ private static List<CustomTypeWithTuple> customTypeWithTupleData = new ArrayList<CustomTypeWithTuple>();
private static List<CustomType> customTypeData = new ArrayList<CustomType>();
+ private static List<NestedCustomType> customNestedTypeData = new ArrayList<NestedCustomType>();
+
+
@BeforeClass
public static void insertCustomData() {
customTypeData.add(new CustomType());
+ customTypeWithTupleData.add(new CustomTypeWithTuple());
+ customNestedTypeData.add(new NestedCustomType());
}
@Test
@@ -127,7 +161,6 @@ public class JoinOperatorTest {
ds1.join(ds2).where(5).equalTo(0);
}
- @Ignore
@Test
public void testJoinKeyExpressions1() {
@@ -137,13 +170,29 @@ public class JoinOperatorTest {
// should work
try {
-// ds1.join(ds2).where("myInt").equalTo("myInt");
+ ds1.join(ds2).where("myInt").equalTo("myInt");
+ } catch(Exception e) {
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyExpressionsNested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<NestedCustomType> ds1 = env.fromCollection(customNestedTypeData);
+ DataSet<NestedCustomType> ds2 = env.fromCollection(customNestedTypeData);
+
+ // should work
+ try {
+ ds1.join(ds2).where("myInt").equalTo("myInt");
} catch(Exception e) {
Assert.fail();
}
}
+
+
- @Ignore
@Test(expected = InvalidProgramException.class)
public void testJoinKeyExpressions2() {
@@ -152,10 +201,9 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible join key types
-// ds1.join(ds2).where("myInt").equalTo("myString");
+ ds1.join(ds2).where("myInt").equalTo("myString");
}
- @Ignore
@Test(expected = InvalidProgramException.class)
public void testJoinKeyExpressions3() {
@@ -164,10 +212,9 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible number of join keys
-// ds1.join(ds2).where("myInt", "myString").equalTo("myString");
+ ds1.join(ds2).where("myInt", "myString").equalTo("myString");
}
- @Ignore
@Test(expected = IllegalArgumentException.class)
public void testJoinKeyExpressions4() {
@@ -176,7 +223,230 @@ public class JoinOperatorTest {
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, join key non-existent
-// ds1.join(ds2).where("myNonExistent").equalTo("myInt");
+ ds1.join(ds2).where("myNonExistent").equalTo("myInt");
+ }
+
+ /**
+ * Test if mixed types of key selectors are properly working.
+ */
+ @Test
+ public void testJoinKeyMixedKeySelector() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+ try {
+ ds1.join(ds2).where("myInt").equalTo(new KeySelector<CustomType, Integer>() {
+ @Override
+ public Integer getKey(CustomType value) throws Exception {
+ return value.myInt;
+ }
+ });
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyMixedKeySelectorTurned() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+ try {
+ ds1.join(ds2).where(new KeySelector<CustomType, Integer>() {
+ @Override
+ public Integer getKey(CustomType value) throws Exception {
+ return value.myInt;
+ }
+ }).equalTo("myInt");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyMixedTupleIndex() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ try {
+ ds1.join(ds2).where("f0").equalTo(4);
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyNestedTuples() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyNestedTupleData, nestedTupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ try {
+ ds1.join(ds2).where("f0.f0").equalTo(4);
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyNestedTuplesWithCustom() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<CustomType, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyNestedCustomTupleData, nestedCustomTupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ try {
+ TypeInformation<?> t = ds1.join(ds2).where("f0.myInt").equalTo(4).getType();
+ Assert.assertTrue("not a composite type", t instanceof CompositeType);
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyWithCustomContainingTuple0() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomTypeWithTuple> ds1 = env.fromCollection(customTypeWithTupleData);
+ DataSet<CustomTypeWithTuple> ds2 = env.fromCollection(customTypeWithTupleData);
+ try {
+ ds1.join(ds2).where("intByString.f0").equalTo("myInt");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyWithCustomContainingTuple1() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomTypeWithTuple> ds1 = env.fromCollection(customTypeWithTupleData);
+ DataSet<CustomTypeWithTuple> ds2 = env.fromCollection(customTypeWithTupleData);
+ try {
+ ds1.join(ds2).where("nested.myInt").equalTo("intByString.f0");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testJoinKeyWithCustomContainingTuple2() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomTypeWithTuple> ds1 = env.fromCollection(customTypeWithTupleData);
+ DataSet<CustomTypeWithTuple> ds2 = env.fromCollection(customTypeWithTupleData);
+ try {
+ ds1.join(ds2).where("nested.myInt", "myInt", "intByString.f1").equalTo("intByString.f0","myInt", "myString");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testJoinKeyNestedTuplesWrongType() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Tuple2<Integer, String>, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyNestedTupleData, nestedTupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ ds1.join(ds2).where("f0.f1").equalTo(4); // f0.f1 is a String
+ }
+
+ @Test
+ public void testJoinKeyMixedTupleIndexTurned() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ try {
+ ds1.join(ds2).where(0).equalTo("f0");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testJoinKeyMixedTupleIndexWrongType() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ ds1.join(ds2).where("f0").equalTo(3); // 3 is of type long, so it should fail
+ }
+
+ @Test
+ public void testJoinKeyMixedTupleIndex2() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
+ try {
+ ds1.join(ds2).where("myInt").equalTo(4);
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testJoinKeyMixedWrong() {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+ // wrongly mix String and Integer
+ ds1.join(ds2).where("myString").equalTo(new KeySelector<CustomType, Integer>() {
+ @Override
+ public Integer getKey(CustomType value) throws Exception {
+ return value.myInt;
+ }
+ });
+ }
+
+ @Test
+ public void testJoinKeyExpressions1Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should work
+ try {
+ ds1.join(ds2).where("nested.myInt").equalTo("nested.myInt");
+ } catch(Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testJoinKeyExpressions2Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, incompatible join key types
+ ds1.join(ds2).where("nested.myInt").equalTo("nested.myString");
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testJoinKeyExpressions3Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, incompatible number of join keys
+ ds1.join(ds2).where("nested.myInt", "nested.myString").equalTo("nested.myString");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testJoinKeyExpressions4Nested() {
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
+ DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
+
+ // should not work, join key non-existent
+ ds1.join(ds2).where("nested.myNonExistent").equalTo("nested.myInt");
}
@@ -235,6 +505,7 @@ public class JoinOperatorTest {
)
.equalTo(3);
} catch(Exception e) {
+ e.printStackTrace();
Assert.fail();
}
}
@@ -403,7 +674,6 @@ public class JoinOperatorTest {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
-
// should work
try {
ds1.join(ds2)
@@ -430,6 +700,7 @@ public class JoinOperatorTest {
.types(CustomType.class, CustomType.class);
} catch(Exception e) {
System.out.println("FAILED: " + e);
+ e.printStackTrace();
Assert.fail();
}
}
@@ -550,13 +821,58 @@ public class JoinOperatorTest {
* ####################################################################
*/
+ public static class Nested implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public int myInt;
+
+ public Nested() {};
+
+ public Nested(int i, long l, String s) {
+ myInt = i;
+ }
+
+ @Override
+ public String toString() {
+ return ""+myInt;
+ }
+ }
+ // a simple nested type (only basic types)
+ public static class NestedCustomType implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public int myInt;
+ public long myLong;
+ public String myString;
+ public Nested nest;
+
+ public NestedCustomType() {};
+
+ public NestedCustomType(int i, long l, String s) {
+ myInt = i;
+ myLong = l;
+ myString = s;
+ }
+
+ @Override
+ public String toString() {
+ return myInt+","+myLong+","+myString+","+nest;
+ }
+ }
+
public static class CustomType implements Serializable {
private static final long serialVersionUID = 1L;
public int myInt;
public long myLong;
+ public NestedCustomType nested;
public String myString;
+ public Object nothing;
+ // public List<String> countries; need Kryo to support this
+ // public Writable interfaceTest; need kryo
public CustomType() {};
@@ -564,6 +880,7 @@ public class JoinOperatorTest {
myInt = i;
myLong = l;
myString = s;
+ nested = new NestedCustomType(i, l, s);
}
@Override
@@ -571,4 +888,33 @@ public class JoinOperatorTest {
return myInt+","+myLong+","+myString;
}
}
+
+
+ public static class CustomTypeWithTuple implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public int myInt;
+ public long myLong;
+ public NestedCustomType nested;
+ public String myString;
+ public Tuple2<Integer, String> intByString;
+
+ public CustomTypeWithTuple() {};
+
+ public CustomTypeWithTuple(int i, long l, String s) {
+ myInt = i;
+ myLong = l;
+ myString = s;
+ nested = new NestedCustomType(i, l, s);
+ intByString = new Tuple2<Integer, String>(i, s);
+ }
+
+ @Override
+ public String toString() {
+ return myInt+","+myLong+","+myString;
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
new file mode 100644
index 0000000..00ab520
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.operators;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
+
+@RunWith(PowerMockRunner.class)
+public class KeysTest {
+
+ @Test
+ public void testTupleRangeCheck() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
+
+ // test private static final int[] rangeCheckFields(int[] fields, int maxAllowedField)
+ Method rangeCheckFieldsMethod = Whitebox.getMethod(Keys.class, "rangeCheckFields", int[].class, int.class);
+ int[] result = (int[]) rangeCheckFieldsMethod.invoke(null, new int[] {1,2,3,4}, 4);
+ Assert.assertArrayEquals(new int[] {1,2,3,4}, result);
+
+ // test duplicate elimination
+ result = (int[]) rangeCheckFieldsMethod.invoke(null, new int[] {1,2,2,3,4}, 4);
+ Assert.assertArrayEquals(new int[] {1,2,3,4}, result);
+
+ result = (int[]) rangeCheckFieldsMethod.invoke(null, new int[] {1,2,2,2,2,2,2,3,3,4}, 4);
+ Assert.assertArrayEquals(new int[] {1,2,3,4}, result);
+
+ // corner case tests
+ result = (int[]) rangeCheckFieldsMethod.invoke(null, new int[] {0}, 0);
+ Assert.assertArrayEquals(new int[] {0}, result);
+
+ Throwable ex = null;
+ try {
+ // throws illegal argument.
+ result = (int[]) rangeCheckFieldsMethod.invoke(null, new int[] {5}, 0);
+ } catch(Throwable iae) {
+ ex = iae;
+ }
+ Assert.assertNotNull(ex);
+ }
+
+ @Test
+ public void testStandardTupleKeys() {
+ TupleTypeInfo<Tuple7<String, String, String, String, String, String, String>> typeInfo = new TupleTypeInfo<Tuple7<String, String, String, String, String, String, String>>(
+ BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO);
+
+ ExpressionKeys<Tuple7<String, String, String, String, String, String, String>> ek;
+
+ for( int i = 1; i < 8; i++) {
+ int[] ints = new int[i];
+ for( int j = 0; j < i; j++) {
+ ints[j] = j;
+ }
+ int[] inInts = Arrays.copyOf(ints, ints.length); // copy, just to make sure that the code is not cheating by changing the ints.
+ ek = new ExpressionKeys<Tuple7<String, String, String, String, String, String, String>>(inInts, typeInfo);
+ Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions());
+ Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length);
+
+ ArrayUtils.reverse(ints);
+ inInts = Arrays.copyOf(ints, ints.length);
+ ek = new ExpressionKeys<Tuple7<String, String, String, String, String, String, String>>(inInts, typeInfo);
+ Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions());
+ Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length);
+ }
+ }
+
+ @Test
+ public void testInvalid() throws Throwable {
+ TupleTypeInfo<Tuple3<String, Tuple3<String, String, String>, String>> typeInfo = new TupleTypeInfo<Tuple3<String,Tuple3<String,String,String>,String>>(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ new TupleTypeInfo<Tuple3<String, String, String>>(BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO);
+ ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>> fpk;
+
+ String[][] tests = new String[][] {
+ new String[] {"f11"},new String[] {"f-35"}, new String[] {"f0.f33"}, new String[] {"f1.f33"}
+ };
+ for(int i = 0; i < tests.length; i++) {
+ Throwable e = null;
+ try {
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(tests[i], typeInfo);
+ } catch(Throwable t) {
+ // System.err.println("Message: "+t.getMessage()); t.printStackTrace();
+ e = t;
+ }
+ Assert.assertNotNull(e);
+ }
+ }
+
+ @Test
+ public void testTupleKeyExpansion() {
+ TupleTypeInfo<Tuple3<String, Tuple3<String, String, String>, String>> typeInfo = new TupleTypeInfo<Tuple3<String,Tuple3<String,String,String>,String>>(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ new TupleTypeInfo<Tuple3<String, String, String>>(BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO);
+ ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>> fpk =
+ new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new int[] {0}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new int[] {1}, typeInfo);
+ Assert.assertArrayEquals(new int[] {1,2,3}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new int[] {2}, typeInfo);
+ Assert.assertArrayEquals(new int[] {4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new int[] {0,1,2}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(null, typeInfo, true); // empty case
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
+ // duplicate elimination
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new int[] {0,1,1,1,2}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"*"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
+ // this was a bug:
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f2"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f0","f1.f0","f1.f1", "f1.f2", "f2"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f0","f1.f0","f1.f1", "f2"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,4}, fpk.computeLogicalKeyPositions());
+
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f2", "f0"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {4,0}, fpk.computeLogicalKeyPositions());
+
+ // duplicate elimination
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f2","f2","f2", "f0"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {4,0}, fpk.computeLogicalKeyPositions());
+
+
+ TupleTypeInfo<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>> complexTypeInfo = new TupleTypeInfo<Tuple3<String,Tuple3<Tuple3<String, String, String>,String,String>,String>>(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ new TupleTypeInfo<Tuple3<Tuple3<String, String, String>, String, String>>(new TupleTypeInfo<Tuple3<String, String, String>>(BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO),BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO);
+
+ ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>> complexFpk =
+ new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new int[] {0}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {0}, complexFpk.computeLogicalKeyPositions());
+
+ complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new int[] {0,1,2}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions());
+
+ complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"*"}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions());
+
+ complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"f1.f0.*"}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {1,2,3}, complexFpk.computeLogicalKeyPositions());
+
+ complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"f2"}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {6}, complexFpk.computeLogicalKeyPositions());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
index 62f1fad..c1cb249 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeInformationTest.java
@@ -22,82 +22,81 @@ package org.apache.flink.api.java.type.extractor;
import static org.junit.Assert.assertTrue;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.junit.Ignore;
+import org.junit.Assert;
import org.junit.Test;
@SuppressWarnings("unused")
public class PojoTypeInformationTest {
- static class SimplePojo {
- String str;
- Boolean Bl;
- boolean bl;
- Byte Bt;
- byte bt;
- Short Shrt;
- short shrt;
- Integer Intgr;
- int intgr;
- Long Lng;
- long lng;
- Float Flt;
- float flt;
- Double Dbl;
- double dbl;
- Character Ch;
- char ch;
- int[] primIntArray;
- Integer[] intWrapperArray;
+ public static class SimplePojo {
+ public String str;
+ public Boolean Bl;
+ public boolean bl;
+ public Byte Bt;
+ public byte bt;
+ public Short Shrt;
+ public short shrt;
+ public Integer Intgr;
+ public int intgr;
+ public Long Lng;
+ public long lng;
+ public Float Flt;
+ public float flt;
+ public Double Dbl;
+ public double dbl;
+ public Character Ch;
+ public char ch;
+ public int[] primIntArray;
+ public Integer[] intWrapperArray;
}
- @Ignore
@Test
public void testSimplePojoTypeExtraction() {
TypeInformation<SimplePojo> type = TypeExtractor.getForClass(SimplePojo.class);
- assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
+ assertTrue("Extracted type is not a composite/pojo type but should be.", type instanceof CompositeType);
}
- static class NestedPojoInner {
- private String field;
+ public static class NestedPojoInner {
+ public String field;
}
- static class NestedPojoOuter {
- private Integer intField;
- NestedPojoInner inner;
+ public static class NestedPojoOuter {
+ public Integer intField;
+ public NestedPojoInner inner;
}
- @Ignore
@Test
public void testNestedPojoTypeExtraction() {
TypeInformation<NestedPojoOuter> type = TypeExtractor.getForClass(NestedPojoOuter.class);
- assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
+ assertTrue("Extracted type is not a Pojo type but should be.", type instanceof CompositeType);
}
- static class Recursive1Pojo {
- private Integer intField;
- Recursive2Pojo rec;
+ public static class Recursive1Pojo {
+ public Integer intField;
+ public Recursive2Pojo rec;
}
- static class Recursive2Pojo {
- private String strField;
- Recursive1Pojo rec;
+ public static class Recursive2Pojo {
+ public String strField;
+ public Recursive1Pojo rec;
}
- @Ignore
@Test
public void testRecursivePojoTypeExtraction() {
// This one tests whether a recursive pojo is detected using the set of visited
// types in the type extractor. The recursive field will be handled using the generic serializer.
TypeInformation<Recursive1Pojo> type = TypeExtractor.getForClass(Recursive1Pojo.class);
- assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
+ assertTrue("Extracted type is not a Pojo type but should be.", type instanceof CompositeType);
}
-
- @Ignore
+
@Test
public void testRecursivePojoObjectTypeExtraction() {
TypeInformation<Recursive1Pojo> type = TypeExtractor.getForObject(new Recursive1Pojo());
- assertTrue("Extracted type is not a Pojo type but should be.", type instanceof PojoTypeInfo);
+ assertTrue("Extracted type is not a Pojo type but should be.", type instanceof CompositeType);
}
+
}
[09/11] git commit: Added expression keys to distinct and partition
operator and addressed some of the comments
Posted by rm...@apache.org.
Added expression keys to distinct and partition operator and addressed some of the comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/aca6fbcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/aca6fbcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/aca6fbcd
Branch: refs/heads/master
Commit: aca6fbcd6034cd3d700734f2c50a1084a9c22eb9
Parents: 926f835
Author: Robert Metzger <me...@web.de>
Authored: Sun Oct 5 19:30:18 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
.../java/org/apache/flink/api/java/DataSet.java | 26 +
.../flink/api/java/SerializationSpeedTest.java | 150 ------
.../api/java/operators/DistinctOperator.java | 5 +-
.../api/java/operators/PartitionOperator.java | 5 +-
.../flink/api/java/typeutils/TypeExtractor.java | 3 +-
.../type/extractor/PojoTypeExtractionTest.java | 488 +++++++++++++++++++
.../java/type/extractor/TypeExtractorTest.java | 456 +----------------
.../runtime/FieldAccessMinibenchmark.java | 150 ++++++
.../typeutils/runtime/PojoComparatorTest.java | 64 +++
.../typeutils/runtime/PojoContainingTuple.java | 44 ++
.../runtime/PojoGenericTypeSerializerTest.java | 32 ++
.../test/javaApiOperators/DistinctITCase.java | 47 +-
.../test/javaApiOperators/PartitionITCase.java | 41 +-
.../util/CollectionDataSets.java | 13 +
14 files changed, 915 insertions(+), 609 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index 424d30b..dd8c4ba 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -434,6 +434,20 @@ public abstract class DataSet<T> {
}
/**
+ * Returns a distinct set of a {@link Tuple} {@link DataSet} using expression keys.
+ * <p/>
+ * The field position keys specify the fields of Tuples or Pojos on which the decision is made if two elements are distinct or
+ * not.
+ * <p/>
+ *
+ * @param fields One or more field positions on which the distinction of the DataSet is decided.
+ * @return A DistinctOperator that represents the distinct DataSet.
+ */
+ public DistinctOperator<T> distinct(String... fields) {
+ return new DistinctOperator<T>(this, new Keys.ExpressionKeys<T>(fields, getType()));
+ }
+
+ /**
* Returns a distinct set of a {@link Tuple} {@link DataSet} using all fields of the tuple.
* <p/>
* Note: This operator can only be applied to Tuple DataSets.
@@ -867,6 +881,18 @@ public abstract class DataSet<T> {
}
/**
+ * Hash-partitions a DataSet on the specified key fields.
+ * <p>
+ * <b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
+ *
+ * @param fields The field expressions on which the DataSet is hash-partitioned.
+ * @return The partitioned DataSet.
+ */
+ public PartitionOperator<T> partitionByHash(String... fields) {
+ return new PartitionOperator<T>(this, PartitionMethod.HASH, new Keys.ExpressionKeys<T>(fields, getType()));
+ }
+
+ /**
* Partitions a DataSet using the specified KeySelector.
* <p>
* <b>Important:</b>This operation shuffles the whole DataSet over the network and can take significant amount of time.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java b/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
deleted file mode 100644
index a81f55d..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/SerializationSpeedTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.java;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Field;
-
-public class SerializationSpeedTest {
-
- static Field wordDescField;
- static Field wordField;
- static {
- try {
- wordDescField = WC.class.getField("wordDesc");
- wordField = ComplexWordDescriptor.class.getField("word");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static class ComplexWordDescriptor {
- public String word;
-
- public String getWord() {
- return word;
- }
- }
-
- public static class WC {
- public int count;
- public ComplexWordDescriptor wordDesc;
-
- public WC(int c, String s) throws NoSuchFieldException,
- SecurityException {
- this.count = c;
- this.wordDesc = new ComplexWordDescriptor();
- this.wordDesc.word = s;
- }
-
- public ComplexWordDescriptor getWordDesc() {
- return wordDesc;
- }
-
- }
-
- public static int compareCodeGenPublicFields(WC w1, WC w2) {
- return w1.wordDesc.word.compareTo(w2.wordDesc.word);
- }
-
- public static int compareCodeGenMethods(WC w1, WC w2) {
- return w1.getWordDesc().getWord().compareTo(w2.getWordDesc().getWord());
- }
-
- public static int compareReflective(WC w1, WC w2)
- throws IllegalArgumentException, IllegalAccessException {
- // get String of w1
- Object wordDesc1 = wordDescField.get(w1);
- String word2cmp1 = (String) wordField.get(wordDesc1);
-
- // get String of w2
- Object wordDesc2 = wordDescField.get(w2);
- String word2cmp2 = (String) wordField.get(wordDesc2);
-
- return word2cmp1.compareTo(word2cmp2);
- }
-
- /**
- * results on Core i7 2600k
- *
- *
- * warming up Code gen 5019 Reflection 20364 Factor = 4.057382
- */
- public static void main(String[] args) throws NoSuchFieldException,
- SecurityException, IllegalArgumentException, IllegalAccessException {
- final long RUNS = 1000000000L;
-
- final RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
- String jvm = bean.getVmName() + " - " + bean.getVmVendor() + " - "
- + bean.getSpecVersion() + '/' + bean.getVmVersion();
- System.err.println("Jvm info : " + jvm);
-
- WC word0 = new WC(14, "Hallo");
- WC word1 = new WC(3, "Hola");
-
- System.err.println("warming up");
- for (long i = 0; i < 100000000; i++) {
- compareCodeGenPublicFields(word0, word1);
- compareCodeGenMethods(word0, word1);
- compareReflective(word0, word1);
- }
-
- System.err.println("Code gen public fields");
- long startTime = System.currentTimeMillis();
- for (long i = 0; i < RUNS; i++) {
- int a = compareCodeGenPublicFields(word0, word1);
- if (a == 0) {
- System.err.println("hah");
- }
- }
- long stopTime = System.currentTimeMillis();
- long elapsedTimeGen = stopTime - startTime;
- System.err.println(elapsedTimeGen);
-
- System.err.println("Code gen methods");
- startTime = System.currentTimeMillis();
- for (long i = 0; i < RUNS; i++) {
- int a = compareCodeGenPublicFields(word0, word1);
- if (a == 0) {
- System.err.println("hah");
- }
- }
- stopTime = System.currentTimeMillis();
- long elapsedTimeGenMethods = stopTime - startTime;
- System.err.println(elapsedTimeGenMethods);
-
- System.err.println("Reflection");
-
- startTime = System.currentTimeMillis();
- for (long i = 0; i < RUNS; i++) {
- int a = compareReflective(word0, word1);
- if (a == 0) {
- System.err.println("hah");
- }
- }
- stopTime = System.currentTimeMillis();
- long elapsedTimeRef = stopTime - startTime;
- System.err.println(elapsedTimeRef);
-
- System.err.println("Factor vs public = "
- + (elapsedTimeRef / (float) elapsedTimeGen));
- System.err.println("Factor vs methods = "
- + (elapsedTimeRef / (float) elapsedTimeGenMethods));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
index 54a65a9..18fd756 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.GroupReduceOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.functions.RichGroupReduceFunction;
import org.apache.flink.api.common.functions.RichGroupReduceFunction.Combinable;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
@@ -67,8 +68,8 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
// FieldPositionKeys can only be applied on Tuples
- if (keys instanceof Keys.ExpressionKeys && !input.getType().isTupleType()) {
- throw new InvalidProgramException("Distinction on field positions is only possible on tuple data types.");
+ if (keys instanceof Keys.ExpressionKeys && !(input.getType() instanceof CompositeType)) {
+ throw new InvalidProgramException("Distinction on field positions is only possible on composite type DataSets.");
}
this.keys = keys;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index 532e464..c4548fb 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.KeyRemovingMapper;
@@ -50,8 +51,8 @@ public class PartitionOperator<T> extends SingleInputUdfOperator<T, T, Partition
throw new UnsupportedOperationException("Range Partitioning not yet supported");
}
- if(pKeys instanceof Keys.ExpressionKeys<?> && !input.getType().isTupleType()) {
- throw new IllegalArgumentException("Hash Partitioning with key fields only possible on Tuple DataSets");
+ if(pKeys instanceof Keys.ExpressionKeys<?> && !(input.getType() instanceof CompositeType) ) {
+ throw new IllegalArgumentException("Hash Partitioning with key fields only possible on Composite-type DataSets");
}
this.pMethod = pMethod;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index bc92cdd..6231a74 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -399,8 +399,7 @@ public class TypeExtractor {
}
// objects with generics are treated as raw type
else if (t instanceof ParameterizedType) {
- return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), new ArrayList<Type>()); // pass new type hierarchies here because
- // while creating the TH here, we assumed a tuple type.
+ return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), typeHierarchy);
}
// no tuple, no TypeVariable, no generic type
else if (t instanceof Class) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
new file mode 100644
index 0000000..34dbd99
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.type.extractor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.PojoField;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.api.java.typeutils.TypeInfoParserTest.MyWritable;
+import org.apache.flink.api.java.typeutils.WritableTypeInfo;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.HashMultiset;
+
+/**
+ * Pojo Type tests
+ *
+ * A Pojo is a bean-style class with getters, setters and empty ctor
+ * OR a class with all fields public (or for every private field, there has to be a public getter/setter)
+ * everything else is a generic type (that can't be used for field selection)
+ */
+public class PojoTypeExtractionTest {
+
+
+ // test with correct pojo types
+ public static class WC { // is a pojo
+ public ComplexNestedClass complex; // is a pojo
+ private int count; // is a BasicType
+
+ public WC() {
+ }
+ public int getCount() {
+ return count;
+ }
+ public void setCount(int c) {
+ this.count = c;
+ }
+ }
+ public static class ComplexNestedClass { // pojo
+ public static int ignoreStaticField;
+ public transient int ignoreTransientField;
+ public Date date; // generic type
+ public Integer someNumber; // BasicType
+ public float someFloat; // BasicType
+ public Tuple3<Long, Long, String> word; //Tuple Type with three basic types
+ public Object nothing; // generic type
+ public MyWritable hadoopCitizen; // writableType
+ }
+
+ // all public test
+ public static class AllPublic extends ComplexNestedClass {
+ public ArrayList<String> somethingFancy; // generic type
+ public HashMultiset<Integer> fancyIds; // generic type
+ public String[] fancyArray; // generic type
+ }
+
+ public static class ParentSettingGenerics extends PojoWithGenerics<Integer, Long> {
+ public String field3;
+ }
+ public static class PojoWithGenerics<T1, T2> {
+ public int key;
+ public T1 field1;
+ public T2 field2;
+ }
+
+ public static class ComplexHierarchyTop extends ComplexHierarchy<Tuple1<String>> {}
+ public static class ComplexHierarchy<T> extends PojoWithGenerics<FromTuple,T> {}
+
+ // extends from Tuple and adds a field
+ public static class FromTuple extends Tuple3<String, String, Long> {
+ private static final long serialVersionUID = 1L;
+ public int special;
+ }
+
+ public static class IncorrectPojo {
+ private int isPrivate;
+ public int getIsPrivate() {
+ return isPrivate;
+ }
+ // setter is missing (intentional)
+ }
+
+ // correct pojo
+ public static class BeanStylePojo {
+ public String abc;
+ private int field;
+ public int getField() {
+ return this.field;
+ }
+ public void setField(int f) {
+ this.field = f;
+ }
+ }
+ public static class WrongCtorPojo {
+ public int a;
+ public WrongCtorPojo(int a) {
+ this.a = a;
+ }
+ }
+
+ // in this test, the location of the getters and setters is mixed across the type hierarchy.
+ public static class TypedPojoGetterSetterCheck extends GenericPojoGetterSetterCheck<String> {
+ public void setPackageProtected(String in) {
+ this.packageProtected = in;
+ }
+ }
+ public static class GenericPojoGetterSetterCheck<T> {
+ T packageProtected;
+ public T getPackageProtected() {
+ return packageProtected;
+ }
+ }
+
+ @Test
+ public void testIncorrectPojos() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(IncorrectPojo.class);
+ Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
+
+ typeForClass = TypeExtractor.createTypeInfo(WrongCtorPojo.class);
+ Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
+ }
+
+ @Test
+ public void testCorrectPojos() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(BeanStylePojo.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+
+ typeForClass = TypeExtractor.createTypeInfo(TypedPojoGetterSetterCheck.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ }
+
+ @Test
+ public void testPojoWC() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(WC.class);
+ checkWCPojoAsserts(typeForClass);
+
+ WC t = new WC();
+ t.complex = new ComplexNestedClass();
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(t);
+ checkWCPojoAsserts(typeForObject);
+ }
+
+ private void checkWCPojoAsserts(TypeInformation<?> typeInfo) {
+ Assert.assertFalse(typeInfo.isBasicType());
+ Assert.assertFalse(typeInfo.isTupleType());
+ Assert.assertEquals(9, typeInfo.getTotalFields());
+ Assert.assertTrue(typeInfo instanceof PojoTypeInfo);
+ PojoTypeInfo<?> pojoType = (PojoTypeInfo<?>) typeInfo;
+
+ List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
+ String[] fields = {"count","complex.date", "complex.hadoopCitizen", "complex.nothing",
+ "complex.someFloat", "complex.someNumber", "complex.word.f0",
+ "complex.word.f1", "complex.word.f2"};
+ int[] positions = {8,0,1,2,
+ 3,4,5,
+ 6,7};
+ Assert.assertEquals(fields.length, positions.length);
+ for(int i = 0; i < fields.length; i++) {
+ pojoType.getKey(fields[i], 0, ffd);
+ Assert.assertEquals("Too many keys returned", 1, ffd.size());
+ Assert.assertEquals("position of field "+fields[i]+" wrong", positions[i], ffd.get(0).getPosition());
+ ffd.clear();
+ }
+
+ pojoType.getKey("complex.word.*", 0, ffd);
+ Assert.assertEquals(3, ffd.size());
+ // check if it returns 5,6,7
+ for(FlatFieldDescriptor ffdE : ffd) {
+ final int pos = ffdE.getPosition();
+ Assert.assertTrue(pos <= 7 );
+ Assert.assertTrue(5 <= pos );
+ if(pos == 5) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 6) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 7) {
+ Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+
+ pojoType.getKey("complex.*", 0, ffd);
+ Assert.assertEquals(8, ffd.size());
+ // check if it returns 0-7
+ for(FlatFieldDescriptor ffdE : ffd) {
+ final int pos = ffdE.getPosition();
+ Assert.assertTrue(ffdE.getPosition() <= 7 );
+ Assert.assertTrue(0 <= ffdE.getPosition() );
+ if(pos == 0) {
+ Assert.assertEquals(Date.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 1) {
+ Assert.assertEquals(MyWritable.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 2) {
+ Assert.assertEquals(Object.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 3) {
+ Assert.assertEquals(Float.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 4) {
+ Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 5) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 6) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 7) {
+ Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+ pojoType.getKey("*", 0, ffd);
+ Assert.assertEquals(9, ffd.size());
+ // check if it returns 0-8
+ for(FlatFieldDescriptor ffdE : ffd) {
+ Assert.assertTrue(ffdE.getPosition() <= 8 );
+ Assert.assertTrue(0 <= ffdE.getPosition() );
+ if(ffdE.getPosition() == 8) {
+ Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+ TypeInformation<?> typeComplexNested = pojoType.getTypeAt(0); // ComplexNestedClass complex
+ Assert.assertTrue(typeComplexNested instanceof PojoTypeInfo);
+
+ Assert.assertEquals(6, typeComplexNested.getArity());
+ Assert.assertEquals(8, typeComplexNested.getTotalFields());
+ PojoTypeInfo<?> pojoTypeComplexNested = (PojoTypeInfo<?>) typeComplexNested;
+
+ boolean dateSeen = false, intSeen = false, floatSeen = false,
+ tupleSeen = false, objectSeen = false, writableSeen = false;
+ for(int i = 0; i < pojoTypeComplexNested.getArity(); i++) {
+ PojoField field = pojoTypeComplexNested.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("date")) {
+ if(dateSeen) {
+ Assert.fail("already seen");
+ }
+ dateSeen = true;
+ Assert.assertEquals(new GenericTypeInfo<Date>(Date.class), field.type);
+ Assert.assertEquals(Date.class, field.type.getTypeClass());
+ } else if(name.equals("someNumber")) {
+ if(intSeen) {
+ Assert.fail("already seen");
+ }
+ intSeen = true;
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ Assert.assertEquals(Integer.class, field.type.getTypeClass());
+ } else if(name.equals("someFloat")) {
+ if(floatSeen) {
+ Assert.fail("already seen");
+ }
+ floatSeen = true;
+ Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, field.type);
+ Assert.assertEquals(Float.class, field.type.getTypeClass());
+ } else if(name.equals("word")) {
+ if(tupleSeen) {
+ Assert.fail("already seen");
+ }
+ tupleSeen = true;
+ Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
+ Assert.assertEquals(Tuple3.class, field.type.getTypeClass());
+ // do some more advanced checks on the tuple
+ TupleTypeInfo<?> tupleTypeFromComplexNested = (TupleTypeInfo<?>) field.type;
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(0));
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(1));
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(2));
+ } else if(name.equals("nothing")) {
+ if(objectSeen) {
+ Assert.fail("already seen");
+ }
+ objectSeen = true;
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ Assert.assertEquals(Object.class, field.type.getTypeClass());
+ } else if(name.equals("hadoopCitizen")) {
+ if(writableSeen) {
+ Assert.fail("already seen");
+ }
+ writableSeen = true;
+ Assert.assertEquals(new WritableTypeInfo<MyWritable>(MyWritable.class), field.type);
+ Assert.assertEquals(MyWritable.class, field.type.getTypeClass());
+ } else {
+ Assert.fail("field "+field+" is not expected");
+ }
+ }
+ Assert.assertTrue("Field was not present", dateSeen);
+ Assert.assertTrue("Field was not present", intSeen);
+ Assert.assertTrue("Field was not present", floatSeen);
+ Assert.assertTrue("Field was not present", tupleSeen);
+ Assert.assertTrue("Field was not present", objectSeen);
+ Assert.assertTrue("Field was not present", writableSeen);
+
+ TypeInformation<?> typeAtOne = pojoType.getTypeAt(1); // int count
+ Assert.assertTrue(typeAtOne instanceof BasicTypeInfo);
+
+ Assert.assertEquals(typeInfo.getTypeClass(), WC.class);
+ Assert.assertEquals(typeInfo.getArity(), 2);
+ }
+
+ @Test
+ public void testPojoAllPublic() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(AllPublic.class);
+ checkAllPublicAsserts(typeForClass);
+
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(new AllPublic() );
+ checkAllPublicAsserts(typeForObject);
+ }
+
+ private void checkAllPublicAsserts(TypeInformation<?> typeInformation) {
+ Assert.assertTrue(typeInformation instanceof PojoTypeInfo);
+ Assert.assertEquals(9, typeInformation.getArity());
+ Assert.assertEquals(11, typeInformation.getTotalFields());
+ // check if the three additional fields are identified correctly
+ boolean arrayListSeen = false, multisetSeen = false, strArraySeen = false;
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("somethingFancy")) {
+ if(arrayListSeen) {
+ Assert.fail("already seen");
+ }
+ arrayListSeen = true;
+ Assert.assertTrue(field.type instanceof GenericTypeInfo);
+ Assert.assertEquals(ArrayList.class, field.type.getTypeClass());
+ } else if(name.equals("fancyIds")) {
+ if(multisetSeen) {
+ Assert.fail("already seen");
+ }
+ multisetSeen = true;
+ Assert.assertTrue(field.type instanceof GenericTypeInfo);
+ Assert.assertEquals(HashMultiset.class, field.type.getTypeClass());
+ } else if(name.equals("fancyArray")) {
+ if(strArraySeen) {
+ Assert.fail("already seen");
+ }
+ strArraySeen = true;
+ Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, field.type);
+ Assert.assertEquals(String[].class, field.type.getTypeClass());
+ } else if(Arrays.asList("date", "someNumber", "someFloat", "word", "nothing", "hadoopCitizen").contains(name)) {
+ // ignore these, they are inherited from the ComplexNestedClass
+ }
+ else {
+ Assert.fail("field "+field+" is not expected");
+ }
+ }
+ Assert.assertTrue("Field was not present", arrayListSeen);
+ Assert.assertTrue("Field was not present", multisetSeen);
+ Assert.assertTrue("Field was not present", strArraySeen);
+ }
+
+ @Test
+ public void testPojoExtendingTuple() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(FromTuple.class);
+ checkFromTuplePojo(typeForClass);
+
+ FromTuple ft = new FromTuple();
+ ft.f0 = ""; ft.f1 = ""; ft.f2 = 0L;
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(ft);
+ checkFromTuplePojo(typeForObject);
+ }
+
+ private void checkFromTuplePojo(TypeInformation<?> typeInformation) {
+ Assert.assertTrue(typeInformation instanceof PojoTypeInfo<?>);
+ Assert.assertEquals(4, typeInformation.getTotalFields());
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("special")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else if(name.equals("f0") || name.equals("f1")) {
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
+ } else if(name.equals("f2")) {
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("unexpected field");
+ }
+ }
+ }
+
+ @Test
+ public void testPojoWithGenerics() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ParentSettingGenerics.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else if (name.equals("field2")) {
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
+ } else if (name.equals("field3")) {
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+ /**
+ * Test if the TypeExtractor is accepting untyped generics,
+ * making them GenericTypes
+ */
+ @Test
+ @Ignore // kryo needed.
+ public void testPojoWithGenericsSomeFieldsGeneric() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(PojoWithGenerics.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ } else if (name.equals("field2")) {
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+
+ @Test
+ public void testPojoWithComplexHierarchy() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ComplexHierarchyTop.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertTrue(field.type instanceof PojoTypeInfo<?>); // From tuple is pojo (not tuple type!)
+ } else if (name.equals("field2")) {
+ Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
+ Assert.assertTrue( ((TupleTypeInfo<?>)field.type).getTypeAt(0).equals(BasicTypeInfo.STRING_TYPE_INFO) );
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index 60d41d3..a092a53 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -22,8 +22,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
import java.util.List;
import org.apache.flink.api.common.functions.InvalidTypesException;
@@ -46,9 +44,7 @@ import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple9;
-import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
-import org.apache.flink.api.java.typeutils.PojoField;
import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -67,7 +63,6 @@ import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
-import com.google.common.collect.HashMultiset;
public class TypeExtractorTest {
@@ -350,449 +345,7 @@ public class TypeExtractorTest {
Assert.assertEquals(ti2.getTypeClass(), CustomType.class);
}
- //
- // Pojo Type tests
- // A Pojo is a bean-style class with getters, setters and empty ctor
- // OR a class with all fields public (or for every private field, there has to be a public getter/setter)
- // everything else is a generic type (that can't be used for field selection)
- //
-
-
- // test with correct pojo types
- public static class WC { // is a pojo
- public ComplexNestedClass complex; // is a pojo
- private int count; // is a BasicType
-
- public WC() {
- }
- public int getCount() {
- return count;
- }
- public void setCount(int c) {
- this.count = c;
- }
- }
- public static class ComplexNestedClass { // pojo
- public static int ignoreStaticField;
- public transient int ignoreTransientField;
- public Date date; // generic type
- public Integer someNumber; // BasicType
- public float someFloat; // BasicType
- public Tuple3<Long, Long, String> word; //Tuple Type with three basic types
- public Object nothing; // generic type
- public MyWritable hadoopCitizen; // writableType
- }
- // all public test
- public static class AllPublic extends ComplexNestedClass {
- public ArrayList<String> somethingFancy; // generic type
- public HashMultiset<Integer> fancyIds; // generic type
- public String[] fancyArray; // generic type
- }
-
- public static class ParentSettingGenerics extends PojoWithGenerics<Integer, Long> {
- public String field3;
- }
- public static class PojoWithGenerics<T1, T2> {
- public int key;
- public T1 field1;
- public T2 field2;
- }
-
- public static class ComplexHierarchyTop extends ComplexHierarchy<Tuple1<String>> {}
- public static class ComplexHierarchy<T> extends PojoWithGenerics<FromTuple,T> {}
-
- // extends from Tuple and adds a field
- public static class FromTuple extends Tuple3<String, String, Long> {
- private static final long serialVersionUID = 1L;
- public int special;
- }
-
- public static class IncorrectPojo {
- private int isPrivate;
- public int getIsPrivate() {
- return isPrivate;
- }
- // setter is missing (intentional)
- }
-
- // correct pojo
- public static class BeanStylePojo {
- public String abc;
- private int field;
- public int getField() {
- return this.field;
- }
- public void setField(int f) {
- this.field = f;
- }
- }
- public static class WrongCtorPojo {
- public int a;
- public WrongCtorPojo(int a) {
- this.a = a;
- }
- }
-
- // in this test, the location of the getters and setters is mixed across the type hierarchy.
- public static class TypedPojoGetterSetterCheck extends GenericPojoGetterSetterCheck<String> {
- public void setPackageProtected(String in) {
- this.packageProtected = in;
- }
- }
- public static class GenericPojoGetterSetterCheck<T> {
- T packageProtected;
- public T getPackageProtected() {
- return packageProtected;
- }
- }
-
- @Test
- public void testIncorrectPojos() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(IncorrectPojo.class);
- Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
-
- typeForClass = TypeExtractor.createTypeInfo(WrongCtorPojo.class);
- Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
- }
-
- @Test
- public void testCorrectPojos() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(BeanStylePojo.class);
- Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
-
- typeForClass = TypeExtractor.createTypeInfo(TypedPojoGetterSetterCheck.class);
- Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
- }
-
- @Test
- public void testPojoWC() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(WC.class);
- checkWCPojoAsserts(typeForClass);
-
- WC t = new WC();
- t.complex = new ComplexNestedClass();
- TypeInformation<?> typeForObject = TypeExtractor.getForObject(t);
- checkWCPojoAsserts(typeForObject);
- }
-
- private void checkWCPojoAsserts(TypeInformation<?> typeInfo) {
- Assert.assertFalse(typeInfo.isBasicType());
- Assert.assertFalse(typeInfo.isTupleType());
- Assert.assertEquals(9, typeInfo.getTotalFields());
- Assert.assertTrue(typeInfo instanceof PojoTypeInfo);
- PojoTypeInfo<?> pojoType = (PojoTypeInfo<?>) typeInfo;
-
- List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
- String[] fields = {"count","complex.date", "complex.hadoopCitizen", "complex.nothing",
- "complex.someFloat", "complex.someNumber", "complex.word.f0",
- "complex.word.f1", "complex.word.f2"};
- int[] positions = {8,0,1,2,
- 3,4,5,
- 6,7};
- Assert.assertEquals(fields.length, positions.length);
- for(int i = 0; i < fields.length; i++) {
- pojoType.getKey(fields[i], 0, ffd);
- Assert.assertEquals("Too many keys returned", 1, ffd.size());
- Assert.assertEquals("position of field "+fields[i]+" wrong", positions[i], ffd.get(0).getPosition());
- ffd.clear();
- }
-
- pojoType.getKey("complex.word.*", 0, ffd);
- Assert.assertEquals(3, ffd.size());
- // check if it returns 5,6,7
- for(FlatFieldDescriptor ffdE : ffd) {
- final int pos = ffdE.getPosition();
- Assert.assertTrue(pos <= 7 );
- Assert.assertTrue(5 <= pos );
- if(pos == 5) {
- Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
- }
- if(pos == 6) {
- Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
- }
- if(pos == 7) {
- Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
- }
- }
- ffd.clear();
-
-
- pojoType.getKey("complex.*", 0, ffd);
- Assert.assertEquals(8, ffd.size());
- // check if it returns 0-7
- for(FlatFieldDescriptor ffdE : ffd) {
- final int pos = ffdE.getPosition();
- Assert.assertTrue(ffdE.getPosition() <= 7 );
- Assert.assertTrue(0 <= ffdE.getPosition() );
- if(pos == 0) {
- Assert.assertEquals(Date.class, ffdE.getType().getTypeClass());
- }
- if(pos == 1) {
- Assert.assertEquals(MyWritable.class, ffdE.getType().getTypeClass());
- }
- if(pos == 2) {
- Assert.assertEquals(Object.class, ffdE.getType().getTypeClass());
- }
- if(pos == 3) {
- Assert.assertEquals(Float.class, ffdE.getType().getTypeClass());
- }
- if(pos == 4) {
- Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
- }
- if(pos == 5) {
- Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
- }
- if(pos == 6) {
- Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
- }
- if(pos == 7) {
- Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
- }
- }
- ffd.clear();
-
- pojoType.getKey("*", 0, ffd);
- Assert.assertEquals(9, ffd.size());
- // check if it returns 0-8
- for(FlatFieldDescriptor ffdE : ffd) {
- Assert.assertTrue(ffdE.getPosition() <= 8 );
- Assert.assertTrue(0 <= ffdE.getPosition() );
- if(ffdE.getPosition() == 8) {
- Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
- }
- }
- ffd.clear();
-
- TypeInformation<?> typeComplexNested = pojoType.getTypeAt(0); // ComplexNestedClass complex
- Assert.assertTrue(typeComplexNested instanceof PojoTypeInfo);
-
- Assert.assertEquals(6, typeComplexNested.getArity());
- Assert.assertEquals(8, typeComplexNested.getTotalFields());
- PojoTypeInfo<?> pojoTypeComplexNested = (PojoTypeInfo<?>) typeComplexNested;
-
- boolean dateSeen = false, intSeen = false, floatSeen = false,
- tupleSeen = false, objectSeen = false, writableSeen = false;
- for(int i = 0; i < pojoTypeComplexNested.getArity(); i++) {
- PojoField field = pojoTypeComplexNested.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("date")) {
- if(dateSeen) {
- Assert.fail("already seen");
- }
- dateSeen = true;
- Assert.assertEquals(new GenericTypeInfo<Date>(Date.class), field.type);
- Assert.assertEquals(Date.class, field.type.getTypeClass());
- } else if(name.equals("someNumber")) {
- if(intSeen) {
- Assert.fail("already seen");
- }
- intSeen = true;
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- Assert.assertEquals(Integer.class, field.type.getTypeClass());
- } else if(name.equals("someFloat")) {
- if(floatSeen) {
- Assert.fail("already seen");
- }
- floatSeen = true;
- Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, field.type);
- Assert.assertEquals(Float.class, field.type.getTypeClass());
- } else if(name.equals("word")) {
- if(tupleSeen) {
- Assert.fail("already seen");
- }
- tupleSeen = true;
- Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
- Assert.assertEquals(Tuple3.class, field.type.getTypeClass());
- // do some more advanced checks on the tuple
- TupleTypeInfo<?> tupleTypeFromComplexNested = (TupleTypeInfo<?>) field.type;
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(0));
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(1));
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(2));
- } else if(name.equals("nothing")) {
- if(objectSeen) {
- Assert.fail("already seen");
- }
- objectSeen = true;
- Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
- Assert.assertEquals(Object.class, field.type.getTypeClass());
- } else if(name.equals("hadoopCitizen")) {
- if(writableSeen) {
- Assert.fail("already seen");
- }
- writableSeen = true;
- Assert.assertEquals(new WritableTypeInfo<MyWritable>(MyWritable.class), field.type);
- Assert.assertEquals(MyWritable.class, field.type.getTypeClass());
- } else {
- Assert.fail("field "+field+" is not expected");
- }
- }
- Assert.assertTrue("Field was not present", dateSeen);
- Assert.assertTrue("Field was not present", intSeen);
- Assert.assertTrue("Field was not present", floatSeen);
- Assert.assertTrue("Field was not present", tupleSeen);
- Assert.assertTrue("Field was not present", objectSeen);
- Assert.assertTrue("Field was not present", writableSeen);
-
- TypeInformation<?> typeAtOne = pojoType.getTypeAt(1); // int count
- Assert.assertTrue(typeAtOne instanceof BasicTypeInfo);
-
- Assert.assertEquals(typeInfo.getTypeClass(), WC.class);
- Assert.assertEquals(typeInfo.getArity(), 2);
- }
-
- @Test
- public void testPojoAllPublic() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(AllPublic.class);
- checkAllPublicAsserts(typeForClass);
-
- TypeInformation<?> typeForObject = TypeExtractor.getForObject(new AllPublic() );
- checkAllPublicAsserts(typeForObject);
- }
-
- private void checkAllPublicAsserts(TypeInformation<?> typeInformation) {
- Assert.assertTrue(typeInformation instanceof PojoTypeInfo);
- Assert.assertEquals(9, typeInformation.getArity());
- Assert.assertEquals(11, typeInformation.getTotalFields());
- // check if the three additional fields are identified correctly
- boolean arrayListSeen = false, multisetSeen = false, strArraySeen = false;
- PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
- for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
- PojoField field = pojoTypeForClass.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("somethingFancy")) {
- if(arrayListSeen) {
- Assert.fail("already seen");
- }
- arrayListSeen = true;
- Assert.assertTrue(field.type instanceof GenericTypeInfo);
- Assert.assertEquals(ArrayList.class, field.type.getTypeClass());
- } else if(name.equals("fancyIds")) {
- if(multisetSeen) {
- Assert.fail("already seen");
- }
- multisetSeen = true;
- Assert.assertTrue(field.type instanceof GenericTypeInfo);
- Assert.assertEquals(HashMultiset.class, field.type.getTypeClass());
- } else if(name.equals("fancyArray")) {
- if(strArraySeen) {
- Assert.fail("already seen");
- }
- strArraySeen = true;
- Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, field.type);
- Assert.assertEquals(String[].class, field.type.getTypeClass());
- } else if(Arrays.asList("date", "someNumber", "someFloat", "word", "nothing", "hadoopCitizen").contains(name)) {
- // ignore these, they are inherited from the ComplexNestedClass
- }
- else {
- Assert.fail("field "+field+" is not expected");
- }
- }
- Assert.assertTrue("Field was not present", arrayListSeen);
- Assert.assertTrue("Field was not present", multisetSeen);
- Assert.assertTrue("Field was not present", strArraySeen);
- }
-
- @Test
- public void testPojoExtendingTuple() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(FromTuple.class);
- checkFromTuplePojo(typeForClass);
-
- FromTuple ft = new FromTuple();
- ft.f0 = ""; ft.f1 = ""; ft.f2 = 0L;
- TypeInformation<?> typeForObject = TypeExtractor.getForObject(ft);
- checkFromTuplePojo(typeForObject);
- }
-
- private void checkFromTuplePojo(TypeInformation<?> typeInformation) {
- Assert.assertTrue(typeInformation instanceof PojoTypeInfo<?>);
- Assert.assertEquals(4, typeInformation.getTotalFields());
- PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
- for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
- PojoField field = pojoTypeForClass.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("special")) {
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- } else if(name.equals("f0") || name.equals("f1")) {
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
- } else if(name.equals("f2")) {
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
- } else {
- Assert.fail("unexpected field");
- }
- }
- }
-
- @Test
- public void testPojoWithGenerics() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ParentSettingGenerics.class);
- Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
- PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
- for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
- PojoField field = pojoTypeForClass.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("field1")) {
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- } else if (name.equals("field2")) {
- Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
- } else if (name.equals("field3")) {
- Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
- } else if (name.equals("key")) {
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- } else {
- Assert.fail("Unexpected field "+field);
- }
- }
- }
-
- /**
- * Test if the TypeExtractor is accepting untyped generics,
- * making them GenericTypes
- */
- @Test
- @Ignore // kryo needed.
- public void testPojoWithGenericsSomeFieldsGeneric() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(PojoWithGenerics.class);
- Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
- PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
- for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
- PojoField field = pojoTypeForClass.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("field1")) {
- Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
- } else if (name.equals("field2")) {
- Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
- } else if (name.equals("key")) {
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- } else {
- Assert.fail("Unexpected field "+field);
- }
- }
- }
-
-
- @Test
- public void testPojoWithComplexHierarchy() {
- TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ComplexHierarchyTop.class);
- Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
- PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
- for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
- PojoField field = pojoTypeForClass.getPojoFieldAt(i);
- String name = field.field.getName();
- if(name.equals("field1")) {
- Assert.assertTrue(field.type instanceof PojoTypeInfo<?>); // From tuple is pojo (not tuple type!)
- } else if (name.equals("field2")) {
- Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
- Assert.assertTrue( ((TupleTypeInfo<?>)field.type).getTypeAt(0).equals(BasicTypeInfo.STRING_TYPE_INFO) );
- } else if (name.equals("key")) {
- Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
- } else {
- Assert.fail("Unexpected field "+field);
- }
- }
- }
-
- // End of Pojo type tests
public static class CustomType {
public String myField1;
@@ -1691,20 +1244,21 @@ public class TypeExtractorTest {
public T myField;
}
+ public static class InType extends MyObject<String> {}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
@Ignore
public void testParamertizedCustomObject() {
- RichMapFunction<?, ?> function = new RichMapFunction<MyObject<String>, MyObject<String>>() {
+ RichMapFunction<?, ?> function = new RichMapFunction<InType, MyObject<String>>() {
private static final long serialVersionUID = 1L;
@Override
- public MyObject<String> map(MyObject<String> value) throws Exception {
+ public MyObject<String> map(InType value) throws Exception {
return null;
}
};
-
- TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("org.apache.flink.api.java.type.extractor.TypeExtractorTest$MyObject"));
+ TypeInformation<?> inType = TypeExtractor.createTypeInfo(InType.class);
+ TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) inType);
Assert.assertTrue(ti instanceof PojoTypeInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/FieldAccessMinibenchmark.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/FieldAccessMinibenchmark.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/FieldAccessMinibenchmark.java
new file mode 100644
index 0000000..8a0ec82
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/FieldAccessMinibenchmark.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.typeutils.runtime;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Field;
+
+public class FieldAccessMinibenchmark {
+
+ static Field wordDescField;
+ static Field wordField;
+ static {
+ try {
+ wordDescField = WC.class.getField("wordDesc");
+ wordField = ComplexWordDescriptor.class.getField("word");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static class ComplexWordDescriptor {
+ public String word;
+
+ public String getWord() {
+ return word;
+ }
+ }
+
+ public static class WC {
+ public int count;
+ public ComplexWordDescriptor wordDesc;
+
+ public WC(int c, String s) throws NoSuchFieldException,
+ SecurityException {
+ this.count = c;
+ this.wordDesc = new ComplexWordDescriptor();
+ this.wordDesc.word = s;
+ }
+
+ public ComplexWordDescriptor getWordDesc() {
+ return wordDesc;
+ }
+
+ }
+
+ public static int compareCodeGenPublicFields(WC w1, WC w2) {
+ return w1.wordDesc.word.compareTo(w2.wordDesc.word);
+ }
+
+ public static int compareCodeGenMethods(WC w1, WC w2) {
+ return w1.getWordDesc().getWord().compareTo(w2.getWordDesc().getWord());
+ }
+
+ public static int compareReflective(WC w1, WC w2)
+ throws IllegalArgumentException, IllegalAccessException {
+ // get String of w1
+ Object wordDesc1 = wordDescField.get(w1);
+ String word2cmp1 = (String) wordField.get(wordDesc1);
+
+ // get String of w2
+ Object wordDesc2 = wordDescField.get(w2);
+ String word2cmp2 = (String) wordField.get(wordDesc2);
+
+ return word2cmp1.compareTo(word2cmp2);
+ }
+
+ /**
+ * results on Core i7 2600k
+ *
+ *
+ * warming up Code gen 5019 Reflection 20364 Factor = 4.057382
+ */
+ public static void main(String[] args) throws NoSuchFieldException,
+ SecurityException, IllegalArgumentException, IllegalAccessException {
+ final long RUNS = 1000000000L;
+
+ final RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
+ String jvm = bean.getVmName() + " - " + bean.getVmVendor() + " - "
+ + bean.getSpecVersion() + '/' + bean.getVmVersion();
+ System.err.println("Jvm info : " + jvm);
+
+ WC word0 = new WC(14, "Hallo");
+ WC word1 = new WC(3, "Hola");
+
+ System.err.println("warming up");
+ for (long i = 0; i < 100000000; i++) {
+ compareCodeGenPublicFields(word0, word1);
+ compareCodeGenMethods(word0, word1);
+ compareReflective(word0, word1);
+ }
+
+ System.err.println("Code gen public fields");
+ long startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareCodeGenPublicFields(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ long stopTime = System.currentTimeMillis();
+ long elapsedTimeGen = stopTime - startTime;
+ System.err.println(elapsedTimeGen);
+
+ System.err.println("Code gen methods");
+ startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareCodeGenPublicFields(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ stopTime = System.currentTimeMillis();
+ long elapsedTimeGenMethods = stopTime - startTime;
+ System.err.println(elapsedTimeGenMethods);
+
+ System.err.println("Reflection");
+
+ startTime = System.currentTimeMillis();
+ for (long i = 0; i < RUNS; i++) {
+ int a = compareReflective(word0, word1);
+ if (a == 0) {
+ System.err.println("hah");
+ }
+ }
+ stopTime = System.currentTimeMillis();
+ long elapsedTimeRef = stopTime - startTime;
+ System.err.println(elapsedTimeRef);
+
+ System.err.println("Factor vs public = "
+ + (elapsedTimeRef / (float) elapsedTimeGen));
+ System.err.println("Factor vs methods = "
+ + (elapsedTimeRef / (float) elapsedTimeGenMethods));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
new file mode 100644
index 0000000..5cb31ca
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import java.util.Arrays;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.ComparatorTestBase;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.TypeComparator;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.junit.Assert;
+import org.junit.Ignore;
+
+
+@Ignore // TODO
+public class PojoComparatorTest extends ComparatorTestBase<PojoContainingTuple> {
+ TypeInformation<PojoContainingTuple> type = TypeExtractor.getForClass(PojoContainingTuple.class);
+
+ PojoContainingTuple[] data = new PojoContainingTuple[]{
+ new PojoContainingTuple(1, 1L, 1L),
+ new PojoContainingTuple(2, 2L, 2L),
+ new PojoContainingTuple(8519, 85190L, 85190L),
+ new PojoContainingTuple(-51498, 85191L, 85191L),
+ };
+
+ @Override
+ protected TypeComparator<PojoContainingTuple> createComparator(boolean ascending) {
+ Assert.assertTrue(type instanceof CompositeType);
+ CompositeType<PojoContainingTuple> cType = (CompositeType<PojoContainingTuple>) type;
+ ExpressionKeys<PojoContainingTuple> keys = new ExpressionKeys<PojoContainingTuple>(new String[] {"theTuple.*"}, cType);
+ boolean[] orders = new boolean[keys.getNumberOfKeyFields()];
+ Arrays.fill(orders, true);
+ return cType.createComparator(keys.computeLogicalKeyPositions(), orders, 0);
+ }
+
+ @Override
+ protected TypeSerializer<PojoContainingTuple> createSerializer() {
+ return type.createSerializer();
+ }
+
+ @Override
+ protected PojoContainingTuple[] getSortedTestData() {
+ return data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoContainingTuple.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoContainingTuple.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoContainingTuple.java
new file mode 100644
index 0000000..ca17ee0
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoContainingTuple.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+
+/**
+ * This file belongs to the PojoComparatorTest test
+ *
+ */
+public class PojoContainingTuple {
+ public int someInt;
+ public String someString = "abc";
+ public Tuple2<Long, Long> theTuple;
+ public PojoContainingTuple() {}
+ public PojoContainingTuple(int i, long l1, long l2) {
+ someInt = i;
+ theTuple = new Tuple2<Long, Long>(l1, l2);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(obj instanceof PojoContainingTuple) {
+ PojoContainingTuple other = (PojoContainingTuple) obj;
+ return someInt == other.someInt && theTuple.equals(other.theTuple);
+ }
+ return false;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoGenericTypeSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoGenericTypeSerializerTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoGenericTypeSerializerTest.java
new file mode 100644
index 0000000..a176178
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoGenericTypeSerializerTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.typeutils.runtime;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+
+public class PojoGenericTypeSerializerTest extends AbstractGenericTypeSerializerTest {
+
+ @Override
+ protected <T> TypeSerializer<T> createSerializer(Class<T> type) {
+ TypeInformation<T> typeInfo = TypeExtractor.getForClass(type);
+ return typeInfo.createSerializer();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
index 69bab75..54f1b58 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/DistinctITCase.java
@@ -25,6 +25,7 @@ import java.util.LinkedList;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -33,6 +34,7 @@ import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -42,7 +44,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class DistinctITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 6;
+ private static int NUM_PROGRAMS = 8;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -237,6 +239,49 @@ public class DistinctITCase extends JavaProgramTestBase {
"5,2\n" +
"5,3\n";
}
+ case 7: {
+
+ /*
+ * check correctness of distinct on tuples with field expressions
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
+ DataSet<Tuple1<Integer>> reduceDs = ds.union(ds)
+ .distinct("f0").project(0).types(Integer.class);
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1\n" +
+ "2\n";
+
+ }
+ case 8: {
+
+ /*
+ * check correctness of distinct on Pojos
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<POJO> ds = CollectionDataSets.getDuplicatePojoDataSet(env);
+ DataSet<Integer> reduceDs = ds.distinct("nestedPojo.longNumber").map(new MapFunction<CollectionDataSets.POJO, Integer>() {
+ @Override
+ public Integer map(POJO value) throws Exception {
+ return (int) value.nestedPojo.longNumber;
+ }
+ });
+
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "10000\n20000\n30000\n";
+
+ }
default:
throw new IllegalArgumentException("Invalid program id");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
index bf1d404..d3c87fa 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
@@ -36,6 +36,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
+import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.apache.flink.util.Collector;
import org.junit.runner.RunWith;
@@ -45,7 +46,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class PartitionITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 3;
+ private static int NUM_PROGRAMS = 4;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -224,6 +225,29 @@ public class PartitionITCase extends JavaProgramTestBase {
"5\n" +
"6\n";
}
+ case 4: {
+ /*
+ * Test hash partition with key expression
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(3);
+
+ DataSet<POJO> ds = CollectionDataSets.getDuplicatePojoDataSet(env);
+ DataSet<Long> uniqLongs = ds
+ .partitionByHash("nestedPojo.longNumber").setParallelism(4)
+ .mapPartition(new UniqueNestedPojoLongMapper());
+ uniqLongs.writeAsText(resultPath);
+
+ env.execute();
+
+ // return expected result
+ return "10000\n" +
+ "20000\n" +
+ "30000\n";
+ }
+
+
default:
throw new IllegalArgumentException("Invalid program id");
@@ -246,6 +270,21 @@ public class PartitionITCase extends JavaProgramTestBase {
}
}
+ public static class UniqueNestedPojoLongMapper implements MapPartitionFunction<POJO, Long> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void mapPartition(Iterable<POJO> records, Collector<Long> out) throws Exception {
+ HashSet<Long> uniq = new HashSet<Long>();
+ for(POJO t : records) {
+ uniq.add(t.nestedPojo.longNumber);
+ }
+ for(Long l : uniq) {
+ out.collect(l);
+ }
+ }
+ }
+
public static class PartitionIndexMapper extends RichMapFunction<Long, Tuple2<Integer, Integer>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/aca6fbcd/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
index b657545..d023287 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
@@ -278,6 +278,19 @@ public class CollectionDataSets {
return env.fromCollection(data);
}
+ public static DataSet<POJO> getDuplicatePojoDataSet(ExecutionEnvironment env) {
+ List<POJO> data = new ArrayList<POJO>();
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L)); // 5x
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(2, "Second",20, 200, 2000L, "Two", 20000L));
+ data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L)); // 2x
+ data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L));
+ return env.fromCollection(data);
+ }
+
public static class POJO {
public int number;
public String str;
[02/11] git commit: Really add POJO support and nested keys for Scala
API
Posted by rm...@apache.org.
Really add POJO support and nested keys for Scala API
This also adds more integration tests, but not all tests of the Java API
have been ported to Scala yet.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/6be85554
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/6be85554
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/6be85554
Branch: refs/heads/master
Commit: 6be85554332fb576abc16e39d866ed05a7c38b72
Parents: 598ae37
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue Oct 7 12:00:15 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
.../scala/graph/ConnectedComponents.scala | 2 +-
.../apache/flink/api/java/operators/Keys.java | 8 +-
.../type/extractor/PojoTypeExtractionTest.java | 4 +-
.../org/apache/flink/api/scala/DataSet.scala | 15 +-
.../apache/flink/api/scala/GroupedDataSet.scala | 42 +-
.../flink/api/scala/codegen/TypeAnalyzer.scala | 167 ++----
.../api/scala/codegen/TypeDescriptors.scala | 45 +-
.../api/scala/codegen/TypeInformationGen.scala | 75 ++-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 84 ++-
.../api/scala/unfinishedKeyPairOperation.scala | 16 +-
.../api/scala/operators/AggregateITCase.scala | 39 +-
.../scala/operators/CoGroupOperatorTest.scala | 34 +-
.../flink/api/scala/operators/CustomType.scala | 33 --
.../api/scala/operators/DistinctITCase.scala | 191 ++++++
.../scala/operators/DistinctOperatorTest.scala | 20 +-
.../api/scala/operators/ExamplesITCase.scala | 79 ++-
.../api/scala/operators/GroupReduceITCase.scala | 574 +++++++++++++++++++
.../api/scala/operators/GroupingTest.scala | 31 +-
.../api/scala/operators/JoinOperatorTest.scala | 33 +-
.../api/scala/operators/PartitionITCase.scala | 84 +--
.../scala/types/TypeInformationGenTest.scala | 10 +-
.../api/scala/util/CollectionDataSets.scala | 394 +++++++++++++
22 files changed, 1587 insertions(+), 393 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
index 4462e45..d261173 100644
--- a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
+++ b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/graph/ConnectedComponents.scala
@@ -76,7 +76,7 @@ object ConnectedComponents {
val edges = getEdgesDataSet(env).flatMap { edge => Seq(edge, (edge._2, edge._1)) }
// open a delta iteration
- val verticesWithComponents = vertices.iterateDelta(vertices, maxIterations, Array(0)) {
+ val verticesWithComponents = vertices.iterateDelta(vertices, maxIterations, Array("_1")) {
(s, ws) =>
// apply the step logic: join with the edges
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
index 482370e..40ce238 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
+import com.google.common.base.Joiner;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -306,7 +307,12 @@ public abstract class Keys<T> {
}
return Ints.toArray(logicalKeys);
}
-
+
+ @Override
+ public String toString() {
+ Joiner join = Joiner.on('.');
+ return "ExpressionKeys: " + join.join(keyFields);
+ }
}
private static String[] removeDuplicates(String[] in) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
index 01d32c1..83c81f7 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
@@ -345,7 +345,9 @@ public class PojoTypeExtractionTest {
Assert.assertEquals(typeInfo.getTypeClass(), WC.class);
Assert.assertEquals(typeInfo.getArity(), 2);
}
-
+
+ // Kryo is required for this, so disable for now.
+ @Ignore
@Test
public void testPojoAllPublic() {
TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(AllPublic.class);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
index 895b964..7a2c699 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
@@ -550,14 +550,11 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
/**
* Creates a new DataSet containing the distinct elements of this DataSet. The decision whether
* two elements are distinct or not is made based on only the specified fields.
- *
- * This only works on CaseClass DataSets
*/
def distinct(firstField: String, otherFields: String*): DataSet[T] = {
- val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
wrap(new DistinctOperator[T](
javaSet,
- new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType, true)))
+ new Keys.ExpressionKeys[T](firstField +: otherFields.toArray, javaSet.getType)))
}
/**
@@ -615,8 +612,6 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
* This only works on CaseClass DataSets.
*/
def groupBy(firstField: String, otherFields: String*): GroupedDataSet[T] = {
- // val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
-
new GroupedDataSet[T](
this,
new Keys.ExpressionKeys[T](firstField +: otherFields.toArray, javaSet.getType))
@@ -862,10 +857,8 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
*/
def iterateDelta[R: ClassTag](workset: DataSet[R], maxIterations: Int, keyFields: Array[String])(
stepFunction: (DataSet[T], DataSet[R]) => (DataSet[T], DataSet[R])) = {
- val fieldIndices = fieldNames2Indices(javaSet.getType, keyFields)
-
- val key = new ExpressionKeys[T](fieldIndices, javaSet.getType, false)
+ val key = new ExpressionKeys[T](keyFields, javaSet.getType)
val iterativeSet = new DeltaIteration[T, R](
javaSet.getExecutionEnvironment,
javaSet.getType,
@@ -931,12 +924,10 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
* significant amount of time.
*/
def partitionByHash(firstField: String, otherFields: String*): DataSet[T] = {
- val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
-
val op = new PartitionOperator[T](
javaSet,
PartitionMethod.HASH,
- new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType, false))
+ new Keys.ExpressionKeys[T](firstField +: otherFields.toArray, javaSet.getType))
wrap(op)
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
index d715939..e7d8978 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
@@ -47,7 +47,7 @@ class GroupedDataSet[T: ClassTag](
// These are for optional secondary sort. They are only used
// when using a group-at-a-time reduce function.
- private val groupSortKeyPositions = mutable.MutableList[Int]()
+ private val groupSortKeyPositions = mutable.MutableList[Either[Int, String]]()
private val groupSortOrders = mutable.MutableList[Order]()
/**
@@ -64,7 +64,7 @@ class GroupedDataSet[T: ClassTag](
if (field >= set.getType.getArity) {
throw new IllegalArgumentException("Order key out of tuple bounds.")
}
- groupSortKeyPositions += field
+ groupSortKeyPositions += Left(field)
groupSortOrders += order
this
}
@@ -76,9 +76,7 @@ class GroupedDataSet[T: ClassTag](
* This only works on CaseClass DataSets.
*/
def sortGroup(field: String, order: Order): GroupedDataSet[T] = {
- val fieldIndex = fieldNames2Indices(set.getType, Array(field))(0)
-
- groupSortKeyPositions += fieldIndex
+ groupSortKeyPositions += Right(field)
groupSortOrders += order
this
}
@@ -88,14 +86,32 @@ class GroupedDataSet[T: ClassTag](
*/
private def maybeCreateSortedGrouping(): Grouping[T] = {
if (groupSortKeyPositions.length > 0) {
- val grouping = new SortedGrouping[T](
- set.javaSet,
- keys,
- groupSortKeyPositions(0),
- groupSortOrders(0))
+ val grouping = groupSortKeyPositions(0) match {
+ case Left(pos) =>
+ new SortedGrouping[T](
+ set.javaSet,
+ keys,
+ pos,
+ groupSortOrders(0))
+
+ case Right(field) =>
+ new SortedGrouping[T](
+ set.javaSet,
+ keys,
+ field,
+ groupSortOrders(0))
+
+ }
// now manually add the rest of the keys
for (i <- 1 until groupSortKeyPositions.length) {
- grouping.sortGroup(groupSortKeyPositions(i), groupSortOrders(i))
+ groupSortKeyPositions(i) match {
+ case Left(pos) =>
+ grouping.sortGroup(pos, groupSortOrders(i))
+
+ case Right(field) =>
+ grouping.sortGroup(field, groupSortOrders(i))
+
+ }
}
grouping
} else {
@@ -209,7 +225,7 @@ class GroupedDataSet[T: ClassTag](
}
}
wrap(
- new GroupReduceOperator[T, R](createUnsortedGrouping(),
+ new GroupReduceOperator[T, R](maybeCreateSortedGrouping(),
implicitly[TypeInformation[R]], reducer))
}
@@ -227,7 +243,7 @@ class GroupedDataSet[T: ClassTag](
}
}
wrap(
- new GroupReduceOperator[T, R](createUnsortedGrouping(),
+ new GroupReduceOperator[T, R](maybeCreateSortedGrouping(),
implicitly[TypeInformation[R]], reducer))
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeAnalyzer.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeAnalyzer.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeAnalyzer.scala
index f0ba195..3a4deca 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeAnalyzer.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeAnalyzer.scala
@@ -17,7 +17,6 @@
*/
package org.apache.flink.api.scala.codegen
-import scala.Option.option2Iterable
import scala.collection.GenTraversableOnce
import scala.collection.mutable
import scala.reflect.macros.Context
@@ -59,12 +58,17 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
case PrimitiveType(default, wrapper) => PrimitiveDescriptor(id, tpe, default, wrapper)
case BoxedPrimitiveType(default, wrapper, box, unbox) =>
BoxedPrimitiveDescriptor(id, tpe, default, wrapper, box, unbox)
- case ListType(elemTpe, iter) => analyzeList(id, tpe, elemTpe, iter)
+ case ListType(elemTpe, iter) =>
+ analyzeList(id, tpe, elemTpe, iter)
case CaseClassType() => analyzeCaseClass(id, tpe)
- case BaseClassType() => analyzeClassHierarchy(id, tpe)
case ValueType() => ValueDescriptor(id, tpe)
case WritableType() => WritableDescriptor(id, tpe)
- case _ => GenericClassDescriptor(id, tpe)
+ case JavaType() =>
+ // It's a Java Class, let the TypeExtractor deal with it...
+ c.warning(c.enclosingPosition, s"Type $tpe is a java class. Will be analyzed by " +
+ s"TypeExtractor at runtime.")
+ GenericClassDescriptor(id, tpe)
+ case _ => analyzePojo(id, tpe)
}
}
}
@@ -78,110 +82,63 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
case desc => ListDescriptor(id, tpe, iter, desc)
}
- private def analyzeClassHierarchy(id: Int, tpe: Type): UDTDescriptor = {
-
- val tagField = {
- val (intTpe, intDefault, intWrapper) = PrimitiveType.intPrimitive
- FieldAccessor(
- NoSymbol,
- NoSymbol,
- NullaryMethodType(intTpe),
- isBaseField = true,
- PrimitiveDescriptor(cache.newId, intTpe, intDefault, intWrapper))
+ private def analyzePojo(id: Int, tpe: Type): UDTDescriptor = {
+ val immutableFields = tpe.members filter { _.isTerm } map { _.asTerm } filter { _.isVal }
+ if (immutableFields.nonEmpty) {
+ // We don't support POJOs with immutable fields
+ c.warning(
+ c.enclosingPosition,
+ s"Type $tpe is no POJO, has immutable fields: ${immutableFields.mkString(", ")}.")
+ return GenericClassDescriptor(id, tpe)
}
-
- val subTypes = tpe.typeSymbol.asClass.knownDirectSubclasses.toList flatMap { d =>
-
- val dTpe =
- {
- val tArgs = (tpe.typeSymbol.asClass.typeParams, typeArgs(tpe)).zipped.toMap
- val dArgs = d.asClass.typeParams map { dp =>
- val tArg = tArgs.keySet.find { tp =>
- dp == tp.typeSignature.asSeenFrom(d.typeSignature, tpe.typeSymbol).typeSymbol
- }
- tArg map { tArgs(_) } getOrElse dp.typeSignature
- }
- appliedType(d.asType.toType, dArgs)
- }
+ val fields = tpe.members
+ .filter { _.isTerm }
+ .map { _.asTerm }
+ .filter { _.isVar }
+ .filterNot { _.annotations.exists( _.tpe <:< typeOf[scala.transient]) }
- if (dTpe <:< tpe) {
- Some(analyze(dTpe))
- } else {
- None
- }
+ if (fields.isEmpty) {
+ c.warning(c.enclosingPosition, "Type $tpe has no fields that are visible from Scala Type" +
+ " analysis. Falling back to Java Type Analysis (TypeExtractor).")
+ return GenericClassDescriptor(id, tpe)
}
- val errors = subTypes flatMap { _.findByType[UnsupportedDescriptor] }
-
- errors match {
- case _ :: _ =>
- val errorMessage = errors flatMap {
- case UnsupportedDescriptor(_, subType, errs) =>
- errs map { err => "Subtype " + subType + " - " + err }
- }
- UnsupportedDescriptor(id, tpe, errorMessage)
-
- case Nil if subTypes.isEmpty =>
- UnsupportedDescriptor(id, tpe, Seq("No instantiable subtypes found for base class"))
- case Nil =>
- val (tParams, _) = tpe.typeSymbol.asClass.typeParams.zip(typeArgs(tpe)).unzip
- val baseMembers =
- tpe.members filter { f => f.isMethod } filter { f => f.asMethod.isSetter } map {
- f => (f, f.asMethod.setter, f.asMethod.returnType)
- }
-
- val subMembers = subTypes map {
- case BaseClassDescriptor(_, _, getters, _) => getters
- case CaseClassDescriptor(_, _, _, _, getters) => getters
- case _ => Seq()
- }
-
- val baseFields = baseMembers flatMap {
- case (bGetter, bSetter, bTpe) =>
- val accessors = subMembers map {
- _ find { sf =>
- sf.getter.name == bGetter.name &&
- sf.tpe.termSymbol.asMethod.returnType <:< bTpe.termSymbol.asMethod.returnType
- }
- }
- accessors.forall { _.isDefined } match {
- case true =>
- Some(
- FieldAccessor(
- bGetter,
- bSetter,
- bTpe,
- isBaseField = true,
- analyze(bTpe.termSymbol.asMethod.returnType)))
- case false => None
- }
- }
+ // check whether all fields are either: 1. public, 2. have getter/setter
+ val invalidFields = fields filterNot {
+ f =>
+ f.isPublic ||
+ (f.getter != NoSymbol && f.getter.isPublic && f.setter != NoSymbol && f.setter.isPublic)
+ }
- def wireBaseFields(desc: UDTDescriptor): UDTDescriptor = {
+ if (invalidFields.nonEmpty) {
+ c.warning(c.enclosingPosition, s"Type $tpe is no POJO because it has non-public fields '" +
+ s"${invalidFields.mkString(", ")}' that don't have public getters/setters.")
+ return GenericClassDescriptor(id, tpe)
+ }
- def updateField(field: FieldAccessor) = {
- baseFields find { bf => bf.getter.name == field.getter.name } match {
- case Some(FieldAccessor(_, _, _, _, fieldDesc)) =>
- field.copy(isBaseField = true, desc = fieldDesc)
- case None => field
- }
- }
+ // check whether we have a zero-parameter ctor
+ val hasZeroCtor = tpe.declarations exists {
+ case m: MethodSymbol
+ if m.isConstructor && m.paramss.length == 1 && m.paramss(0).length == 0 => true
+ case _ => false
+ }
- desc match {
- case desc @ BaseClassDescriptor(_, _, getters, baseSubTypes) =>
- desc.copy(
- getters = getters map updateField,
- subTypes = baseSubTypes map wireBaseFields)
- case desc @ CaseClassDescriptor(_, _, _, _, getters) =>
- desc.copy(getters = getters map updateField)
- case _ => desc
- }
- }
+ if (!hasZeroCtor) {
+ // We don't support POJOs without zero-paramter ctor
+ c.warning(
+ c.enclosingPosition,
+ s"Class $tpe is no POJO, has no zero-parameters constructor.")
+ return GenericClassDescriptor(id, tpe)
+ }
- BaseClassDescriptor(id, tpe, tagField +: baseFields.toSeq, subTypes map wireBaseFields)
+ val fieldDescriptors = fields map {
+ f =>
+ val fieldTpe = f.getter.asMethod.returnType.asSeenFrom(tpe, tpe.typeSymbol)
+ FieldDescriptor(f.name.toString.trim, f.getter, f.setter, fieldTpe, analyze(fieldTpe))
}
+ PojoDescriptor(id, tpe, fieldDescriptors.toSeq)
}
private def analyzeCaseClass(id: Int, tpe: Type): UDTDescriptor = {
@@ -216,7 +173,7 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
}
val fields = caseFields map {
case (fgetter, fsetter, fTpe) =>
- FieldAccessor(fgetter, fsetter, fTpe, isBaseField = false, analyze(fTpe))
+ FieldDescriptor(fgetter.name.toString.trim, fgetter, fsetter, fTpe, analyze(fTpe))
}
val mutable = enableMutableUDTs && (fields forall { f => f.setter != NoSymbol })
if (mutable) {
@@ -226,8 +183,9 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
case errs @ _ :: _ =>
val msgs = errs flatMap { f =>
(f: @unchecked) match {
- case FieldAccessor(fgetter, _,_,_, UnsupportedDescriptor(_, fTpe, errors)) =>
- errors map { err => "Field " + fgetter.name + ": " + fTpe + " - " + err }
+ case FieldDescriptor(
+ fName, _, _, _, UnsupportedDescriptor(_, fTpe, errors)) =>
+ errors map { err => "Field " + fName + ": " + fTpe + " - " + err }
}
}
UnsupportedDescriptor(id, tpe, msgs)
@@ -296,11 +254,6 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
def unapply(tpe: Type): Boolean = tpe.typeSymbol.asClass.isCaseClass
}
- private object BaseClassType {
- def unapply(tpe: Type): Boolean =
- tpe.typeSymbol.asClass.isAbstractClass && tpe.typeSymbol.asClass.isSealed
- }
-
private object ValueType {
def unapply(tpe: Type): Boolean =
tpe.typeSymbol.asClass.baseClasses exists {
@@ -315,6 +268,10 @@ private[flink] trait TypeAnalyzer[C <: Context] { this: MacroContextHolder[C]
}
}
+ private object JavaType {
+ def unapply(tpe: Type): Boolean = tpe.typeSymbol.asClass.isJava
+ }
+
private class UDTAnalyzerCache {
private val caches = new DynamicVariable[Map[Type, RecursiveDescriptor]](Map())
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeDescriptors.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeDescriptors.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeDescriptors.scala
index 8201a68..66299c7 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeDescriptors.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeDescriptors.scala
@@ -36,10 +36,8 @@ private[flink] trait TypeDescriptors[C <: Context] { this: MacroContextHolder[C]
def canBeKey: Boolean
- def mkRoot: UDTDescriptor = this
-
def flatten: Seq[UDTDescriptor]
- def getters: Seq[FieldAccessor] = Seq()
+ def getters: Seq[FieldDescriptor] = Seq()
def select(member: String): Option[UDTDescriptor] =
getters find { _.getter.name.toString == member } map { _.desc }
@@ -48,7 +46,7 @@ private[flink] trait TypeDescriptors[C <: Context] { this: MacroContextHolder[C]
case Nil => Seq(Some(this))
case head :: tail => getters find { _.getter.name.toString == head } match {
case None => Seq(None)
- case Some(d : FieldAccessor) => d.desc.select(tail)
+ case Some(d : FieldDescriptor) => d.desc.select(tail)
}
}
@@ -60,7 +58,7 @@ private[flink] trait TypeDescriptors[C <: Context] { this: MacroContextHolder[C]
}
def getRecursiveRefs: Seq[UDTDescriptor] =
- findByType[RecursiveDescriptor].flatMap { rd => findById(rd.refId) }.map { _.mkRoot }.distinct
+ findByType[RecursiveDescriptor].flatMap { rd => findById(rd.refId) }.distinct
}
case class GenericClassDescriptor(id: Int, tpe: Type) extends UDTDescriptor {
@@ -116,30 +114,45 @@ private[flink] trait TypeDescriptors[C <: Context] { this: MacroContextHolder[C]
}
}
- case class BaseClassDescriptor(
- id: Int, tpe: Type, override val getters: Seq[FieldAccessor], subTypes: Seq[UDTDescriptor])
+ case class PojoDescriptor(id: Int, tpe: Type, override val getters: Seq[FieldDescriptor])
extends UDTDescriptor {
- override def flatten =
- this +: ((getters flatMap { _.desc.flatten }) ++ (subTypes flatMap { _.flatten }))
+ override val isPrimitiveProduct = getters.nonEmpty && getters.forall(_.desc.isPrimitiveProduct)
+
+ override def flatten = this +: (getters flatMap { _.desc.flatten })
+
override def canBeKey = flatten forall { f => f.canBeKey }
-
+
+ // Hack: ignore the ctorTpe, since two Type instances representing
+ // the same ctor function type don't appear to be considered equal.
+ // Equality of the tpe and ctor fields implies equality of ctorTpe anyway.
+ override def hashCode = (id, tpe, getters).hashCode
+ override def equals(that: Any) = that match {
+ case PojoDescriptor(thatId, thatTpe, thatGetters) =>
+ (id, tpe, getters).equals(
+ thatId, thatTpe, thatGetters)
+ case _ => false
+ }
+
override def select(path: List[String]): Seq[Option[UDTDescriptor]] = path match {
case Nil => getters flatMap { g => g.desc.select(Nil) }
case head :: tail => getters find { _.getter.name.toString == head } match {
case None => Seq(None)
- case Some(d : FieldAccessor) => d.desc.select(tail)
+ case Some(d : FieldDescriptor) => d.desc.select(tail)
}
}
}
case class CaseClassDescriptor(
- id: Int, tpe: Type, mutable: Boolean, ctor: Symbol, override val getters: Seq[FieldAccessor])
+ id: Int,
+ tpe: Type,
+ mutable: Boolean,
+ ctor: Symbol,
+ override val getters: Seq[FieldDescriptor])
extends UDTDescriptor {
override val isPrimitiveProduct = getters.nonEmpty && getters.forall(_.desc.isPrimitiveProduct)
- override def mkRoot = this.copy(getters = getters map { _.copy(isBaseField = false) })
override def flatten = this +: (getters flatMap { _.desc.flatten })
override def canBeKey = flatten forall { f => f.canBeKey }
@@ -159,16 +172,16 @@ private[flink] trait TypeDescriptors[C <: Context] { this: MacroContextHolder[C]
case Nil => getters flatMap { g => g.desc.select(Nil) }
case head :: tail => getters find { _.getter.name.toString == head } match {
case None => Seq(None)
- case Some(d : FieldAccessor) => d.desc.select(tail)
+ case Some(d : FieldDescriptor) => d.desc.select(tail)
}
}
}
- case class FieldAccessor(
+ case class FieldDescriptor(
+ name: String,
getter: Symbol,
setter: Symbol,
tpe: Type,
- isBaseField: Boolean,
desc: UDTDescriptor)
case class RecursiveDescriptor(id: Int, tpe: Type, refId: Int) extends UDTDescriptor {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala
index f6a89d3..0686668 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/codegen/TypeInformationGen.scala
@@ -20,7 +20,6 @@ package org.apache.flink.api.scala.codegen
import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.common.typeutils.TypeSerializer
import org.apache.flink.api.java.typeutils._
@@ -29,6 +28,8 @@ import org.apache.flink.types.Value
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.hadoop.io.Writable
+import scala.collection.JavaConverters._
+
import scala.reflect.macros.Context
private[flink] trait TypeInformationGen[C <: Context] {
@@ -41,7 +42,7 @@ private[flink] trait TypeInformationGen[C <: Context] {
// This is for external calling by TypeUtils.createTypeInfo
def mkTypeInfo[T: c.WeakTypeTag]: c.Expr[TypeInformation[T]] = {
- val desc = getUDTDescriptor(weakTypeOf[T])
+ val desc = getUDTDescriptor(weakTypeTag[T].tpe)
val result: c.Expr[TypeInformation[T]] = mkTypeInfo(desc)(c.WeakTypeTag(desc.tpe))
result
}
@@ -61,6 +62,7 @@ private[flink] trait TypeInformationGen[C <: Context] {
case d : WritableDescriptor =>
mkWritableTypeInfo(d)(c.WeakTypeTag(d.tpe).asInstanceOf[c.WeakTypeTag[Writable]])
.asInstanceOf[c.Expr[TypeInformation[T]]]
+ case pojo: PojoDescriptor => mkPojo(pojo)
case d => mkGenericTypeInfo(d)
}
@@ -96,7 +98,7 @@ private[flink] trait TypeInformationGen[C <: Context] {
def mkListTypeInfo[T: c.WeakTypeTag](desc: ListDescriptor): c.Expr[TypeInformation[T]] = {
val arrayClazz = c.Expr[Class[T]](Literal(Constant(desc.tpe)))
val elementClazz = c.Expr[Class[T]](Literal(Constant(desc.elem.tpe)))
- val elementTypeInfo = mkTypeInfo(desc.elem)
+ val elementTypeInfo = mkTypeInfo(desc.elem)(c.WeakTypeTag(desc.elem.tpe))
desc.elem match {
// special case for string, which in scala is a primitive, but not in java
case p: PrimitiveDescriptor if p.tpe <:< typeOf[String] =>
@@ -115,7 +117,8 @@ private[flink] trait TypeInformationGen[C <: Context] {
reify {
ObjectArrayTypeInfo.getInfoFor(
arrayClazz.splice,
- elementTypeInfo.splice).asInstanceOf[TypeInformation[T]]
+ elementTypeInfo.splice.asInstanceOf[TypeInformation[_]])
+ .asInstanceOf[TypeInformation[T]]
}
}
}
@@ -136,6 +139,35 @@ private[flink] trait TypeInformationGen[C <: Context] {
}
}
+ def mkPojo[T: c.WeakTypeTag](desc: PojoDescriptor): c.Expr[TypeInformation[T]] = {
+ val tpeClazz = c.Expr[Class[T]](Literal(Constant(desc.tpe)))
+ val fieldsTrees = desc.getters map {
+ f =>
+ val name = c.Expr(Literal(Constant(f.name)))
+ val fieldType = mkTypeInfo(f.desc)(c.WeakTypeTag(f.tpe))
+ reify { (name.splice, fieldType.splice) }.tree
+ }
+
+ val fieldsList = c.Expr[List[(String, TypeInformation[_])]](mkList(fieldsTrees.toList))
+
+ reify {
+ val fields = fieldsList.splice
+ val clazz: Class[T] = tpeClazz.splice
+
+ val fieldMap = TypeExtractor.getAllDeclaredFields(clazz).asScala map {
+ f => (f.getName, f)
+ } toMap
+
+ val pojoFields = fields map {
+ case (fName, fTpe) =>
+ new PojoField(fieldMap(fName), fTpe)
+ }
+
+ new PojoTypeInfo(clazz, pojoFields.asJava)
+
+ }
+ }
+
def mkGenericTypeInfo[T: c.WeakTypeTag](desc: UDTDescriptor): c.Expr[TypeInformation[T]] = {
val tpeClazz = c.Expr[Class[T]](Literal(Constant(desc.tpe)))
reify {
@@ -158,39 +190,4 @@ private[flink] trait TypeInformationGen[C <: Context] {
val result = Apply(Select(New(TypeTree(desc.tpe)), nme.CONSTRUCTOR), fields.toList)
c.Expr[T](result)
}
-
-// def mkCaseClassTypeInfo[T: c.WeakTypeTag](
-// desc: CaseClassDescriptor): c.Expr[TypeInformation[T]] = {
-// val tpeClazz = c.Expr[Class[_]](Literal(Constant(desc.tpe)))
-// val caseFields = mkCaseFields(desc)
-// reify {
-// new ScalaTupleTypeInfo[T] {
-// def createSerializer: TypeSerializer[T] = {
-// null
-// }
-//
-// val fields: Map[String, TypeInformation[_]] = caseFields.splice
-// val clazz = tpeClazz.splice
-// }
-// }
-// }
-//
-// private def mkCaseFields(desc: UDTDescriptor): c.Expr[Map[String, TypeInformation[_]]] = {
-// val fields = getFields("_root_", desc).toList map { case (fieldName, fieldDesc) =>
-// val nameTree = c.Expr(Literal(Constant(fieldName)))
-// val fieldTypeInfo = mkTypeInfo(fieldDesc)(c.WeakTypeTag(fieldDesc.tpe))
-// reify { (nameTree.splice, fieldTypeInfo.splice) }.tree
-// }
-//
-// c.Expr(mkMap(fields))
-// }
-//
-// protected def getFields(name: String, desc: UDTDescriptor): Seq[(String, UDTDescriptor)] =
-// desc match {
-// // Flatten product types
-// case CaseClassDescriptor(_, _, _, _, getters) =>
-// getters filterNot { _.isBaseField } flatMap {
-// f => getFields(name + "." + f.getter.name, f.desc) }
-// case _ => Seq((name, desc))
-// }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
index 3e9d4c6..53d1dea 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
@@ -19,8 +19,10 @@ package org.apache.flink.api.scala.typeutils
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeinfo.AtomicType
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.api.common.typeutils.{CompositeType, TypeComparator}
/**
* TypeInformation for Case Classes. Creation and access is different from
@@ -58,16 +60,82 @@ abstract class CaseClassTypeInfo[T <: Product](
override protected def getNewComparator: TypeComparator[T] = {
val finalLogicalKeyFields = logicalKeyFields.take(comparatorHelperIndex)
val finalComparators = fieldComparators.take(comparatorHelperIndex)
- var maxKey: Int = 0
- for (key <- finalLogicalKeyFields) {
- maxKey = Math.max(maxKey, key)
+ val maxKey = finalLogicalKeyFields.max
+
+ // create serializers only up to the last key, fields after that are not needed
+ val fieldSerializers = types.take(maxKey + 1).map(_.createSerializer)
+ new CaseClassComparator[T](finalLogicalKeyFields, finalComparators, fieldSerializers.toArray)
+ }
+
+ override def getKey(
+ fieldExpression: String,
+ offset: Int,
+ result: java.util.List[FlatFieldDescriptor]): Unit = {
+
+ if (fieldExpression == ExpressionKeys.SELECT_ALL_CHAR) {
+ var keyPosition = 0
+ for (tpe <- types) {
+ tpe match {
+ case a: AtomicType[_] =>
+ result.add(new CompositeType.FlatFieldDescriptor(offset + keyPosition, tpe))
+
+ case co: CompositeType[_] =>
+ co.getKey(ExpressionKeys.SELECT_ALL_CHAR, offset + keyPosition, result)
+ keyPosition += co.getTotalFields - 1
+
+ case _ => throw new RuntimeException(s"Unexpected key type: $tpe")
+
+ }
+ keyPosition += 1
+ }
+ return
+ }
+
+ if (fieldExpression == null || fieldExpression.length <= 0) {
+ throw new IllegalArgumentException("Field expression must not be empty.")
}
- val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](maxKey + 1)
- for (i <- 0 to maxKey) {
- fieldSerializers(i) = types(i).createSerializer
+ fieldExpression.split('.').toList match {
+ case headField :: Nil =>
+ var fieldId = 0
+ for (i <- 0 until fieldNames.length) {
+ fieldId += types(i).getTotalFields - 1
+
+ if (fieldNames(i) == headField) {
+ if (fieldTypes(i).isInstanceOf[CompositeType[_]]) {
+ throw new IllegalArgumentException(
+ s"The specified field '$fieldExpression' is refering to a composite type.\n"
+ + s"Either select all elements in this type with the " +
+ s"'${ExpressionKeys.SELECT_ALL_CHAR}' operator or specify a field in" +
+ s" the sub-type")
+ }
+ result.add(new CompositeType.FlatFieldDescriptor(offset + fieldId, fieldTypes(i)))
+ return
+ }
+
+ fieldId += 1
+ }
+ case firstField :: rest =>
+ var fieldId = 0
+ for (i <- 0 until fieldNames.length) {
+
+ if (fieldNames(i) == firstField) {
+ fieldTypes(i) match {
+ case co: CompositeType[_] =>
+ co.getKey(rest.mkString("."), offset + fieldId, result)
+ return
+
+ case _ =>
+ throw new RuntimeException(s"Field ${fieldTypes(i)} is not a composite type.")
+
+ }
+ }
+
+ fieldId += types(i).getTotalFields
+ }
}
- new CaseClassComparator[T](finalLogicalKeyFields, finalComparators, fieldSerializers)
+
+ throw new RuntimeException(s"Unable to find field $fieldExpression in type $this.")
}
override def toString = clazz.getSimpleName + "(" + fieldNames.zip(types).map {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
index 9d9a19f..b2929b9 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
@@ -69,11 +69,9 @@ private[flink] abstract class UnfinishedKeyPairOperation[L, R, O](
* This only works on a CaseClass [[DataSet]].
*/
def where(firstLeftField: String, otherLeftFields: String*) = {
- val fieldIndices = fieldNames2Indices(
- leftInput.getType,
- firstLeftField +: otherLeftFields.toArray)
-
- val leftKey = new ExpressionKeys[L](fieldIndices, leftInput.getType)
+ val leftKey = new ExpressionKeys[L](
+ firstLeftField +: otherLeftFields.toArray,
+ leftInput.getType)
new HalfUnfinishedKeyPairOperation[L, R, O](this, leftKey)
}
@@ -118,11 +116,9 @@ private[flink] class HalfUnfinishedKeyPairOperation[L, R, O](
* This only works on a CaseClass [[DataSet]].
*/
def equalTo(firstRightField: String, otherRightFields: String*): O = {
- val fieldIndices = fieldNames2Indices(
- unfinished.rightInput.getType,
- firstRightField +: otherRightFields.toArray)
-
- val rightKey = new ExpressionKeys[R](fieldIndices, unfinished.rightInput.getType)
+ val rightKey = new ExpressionKeys[R](
+ firstRightField +: otherRightFields.toArray,
+ unfinished.rightInput.getType)
if (!leftKey.areCompatible(rightKey)) {
throw new InvalidProgramException("The types of the key fields do not match. Left: " +
leftKey + " Right: " + rightKey)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
index 631e68a..0e3f2ed 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateITCase.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.operators
import org.apache.flink.api.java.aggregation.Aggregations
import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.configuration.Configuration
import org.apache.flink.test.util.JavaProgramTestBase
import org.junit.runner.RunWith
@@ -34,38 +35,14 @@ import org.apache.flink.api.scala._
object AggregateProgs {
var NUM_PROGRAMS: Int = 3
- val tupleInput = Array(
- (1,1L,"Hi"),
- (2,2L,"Hello"),
- (3,2L,"Hello world"),
- (4,3L,"Hello world, how are you?"),
- (5,3L,"I am fine."),
- (6,3L,"Luke Skywalker"),
- (7,4L,"Comment#1"),
- (8,4L,"Comment#2"),
- (9,4L,"Comment#3"),
- (10,4L,"Comment#4"),
- (11,5L,"Comment#5"),
- (12,5L,"Comment#6"),
- (13,5L,"Comment#7"),
- (14,5L,"Comment#8"),
- (15,5L,"Comment#9"),
- (16,6L,"Comment#10"),
- (17,6L,"Comment#11"),
- (18,6L,"Comment#12"),
- (19,6L,"Comment#13"),
- (20,6L,"Comment#14"),
- (21,6L,"Comment#15")
- )
-
-
def runProgram(progId: Int, resultPath: String): String = {
progId match {
case 1 =>
// Full aggregate
val env = ExecutionEnvironment.getExecutionEnvironment
env.setDegreeOfParallelism(10)
- val ds = env.fromCollection(tupleInput)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
val aggregateDs = ds
.aggregate(Aggregations.SUM,0)
@@ -84,7 +61,7 @@ object AggregateProgs {
case 2 =>
// Grouped aggregate
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
val aggregateDs = ds
.groupBy(1)
@@ -103,7 +80,7 @@ object AggregateProgs {
case 3 =>
// Nested aggregate
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromCollection(tupleInput)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
val aggregateDs = ds
.groupBy(1)
@@ -111,7 +88,7 @@ object AggregateProgs {
.aggregate(Aggregations.MIN, 0)
// Ensure aggregate operator correctly copies other fields
.filter(_._3 != null)
- .map { t => Tuple1(t._1) }
+ .map { t => new Tuple1(t._1) }
aggregateDs.writeAsCsv(resultPath)
@@ -140,7 +117,7 @@ class AggregateITCase(config: Configuration) extends JavaProgramTestBase(config)
}
protected def testProgram(): Unit = {
- expectedResult = AggregateProgs.runProgram(curProgId, resultPath)
+ expectedResult = DistinctProgs.runProgram(curProgId, resultPath)
}
protected override def postSubmit(): Unit = {
@@ -152,7 +129,7 @@ object AggregateITCase {
@Parameters
def getConfigurations: java.util.Collection[Array[AnyRef]] = {
val configs = mutable.MutableList[Array[AnyRef]]()
- for (i <- 1 to AggregateProgs.NUM_PROGRAMS) {
+ for (i <- 1 to DistinctProgs.NUM_PROGRAMS) {
val config = new Configuration()
config.setInteger("ProgramId", i)
configs += Array(config)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
index d962b76..3f0ca5f 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
@@ -17,13 +17,11 @@
*/
package org.apache.flink.api.scala.operators
-import java.io.Serializable
-import org.apache.flink.api.common.InvalidProgramException
import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
import org.junit.Assert
-import org.junit.Ignore
import org.junit.Test
import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
class CoGroupOperatorTest {
@@ -130,7 +128,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where("_1", "_2").equalTo("_3")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testCoGroupKeyFieldNames4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -140,7 +138,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where("_6").equalTo("_1")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testCoGroupKeyFieldNames5(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -150,7 +148,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where("_1").equalTo("bar")
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[RuntimeException])
def testCoGroupKeyFieldNames6(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -160,7 +158,6 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where("_3").equalTo("_1")
}
- @Ignore
@Test
def testCoGroupKeyExpressions1(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
@@ -176,29 +173,26 @@ class CoGroupOperatorTest {
}
}
- @Ignore
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyExpressions2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(customTypeData)
val ds2 = env.fromCollection(customTypeData)
// should not work, incompatible key types
-// ds1.coGroup(ds2).where("i").equalTo("s")
+ ds1.coGroup(ds2).where("myInt").equalTo("myString")
}
- @Ignore
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyExpressions3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(customTypeData)
val ds2 = env.fromCollection(customTypeData)
// should not work, incompatible number of keys
-// ds1.coGroup(ds2).where("i", "s").equalTo("s")
+ ds1.coGroup(ds2).where("myInt", "myString").equalTo("myString")
}
- @Ignore
@Test(expected = classOf[IllegalArgumentException])
def testCoGroupKeyExpressions4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -207,7 +201,7 @@ class CoGroupOperatorTest {
// should not work, key non-existent
-// ds1.coGroup(ds2).where("myNonExistent").equalTo("i")
+ ds1.coGroup(ds2).where("myNonExistent").equalTo("i")
}
@Test
@@ -218,7 +212,7 @@ class CoGroupOperatorTest {
// Should work
try {
- ds1.coGroup(ds2).where { _.l } equalTo { _.l }
+ ds1.coGroup(ds2).where { _.myLong } equalTo { _.myLong }
}
catch {
case e: Exception => Assert.fail()
@@ -233,7 +227,7 @@ class CoGroupOperatorTest {
// Should work
try {
- ds1.coGroup(ds2).where { _.l}.equalTo(3)
+ ds1.coGroup(ds2).where { _.myLong }.equalTo(3)
}
catch {
case e: Exception => Assert.fail()
@@ -248,7 +242,7 @@ class CoGroupOperatorTest {
// Should work
try {
- ds1.coGroup(ds2).where(3).equalTo { _.l }
+ ds1.coGroup(ds2).where(3).equalTo { _.myLong }
}
catch {
case e: Exception => Assert.fail()
@@ -262,7 +256,7 @@ class CoGroupOperatorTest {
val ds2 = env.fromCollection(customTypeData)
// Should not work, incompatible types
- ds1.coGroup(ds2).where(2).equalTo { _.l }
+ ds1.coGroup(ds2).where(2).equalTo { _.myLong }
}
@Test(expected = classOf[IncompatibleKeysException])
@@ -272,7 +266,7 @@ class CoGroupOperatorTest {
val ds2 = env.fromCollection(customTypeData)
// Should not work, more than one field position key
- ds1.coGroup(ds2).where(1, 3).equalTo { _.l }
+ ds1.coGroup(ds2).where(1, 3).equalTo { _.myLong }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CustomType.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CustomType.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CustomType.scala
deleted file mode 100644
index 94627b9..0000000
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CustomType.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.operators
-
-import java.io.Serializable
-
-/**
- * A custom data type that is used by the operator Tests.
- */
-class CustomType(var i:Int, var l: Long, var s: String) extends Serializable {
- def this() {
- this(0, 0, null)
- }
-
- override def toString: String = {
- i + "," + l + "," + s
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
new file mode 100644
index 0000000..855335d
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctITCase.scala
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object DistinctProgs {
+ var NUM_PROGRAMS: Int = 8
+
+
+ def runProgram(progId: Int, resultPath: String): String = {
+ progId match {
+ case 1 =>
+ /*
+ * Check correctness of distinct on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct(0, 1, 2)
+ distinctDs.writeAsCsv(resultPath)
+
+ env.execute()
+
+ // return expected result
+ "1,1,Hi\n" +
+ "2,2,Hello\n" +
+ "3,2,Hello world\n"
+
+ case 2 =>
+ /*
+ * check correctness of distinct on tuples with key field selector with not all fields
+ * selected
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct(0).map(_._1)
+
+ distinctDs.writeAsText(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 3 =>
+ /*
+ * check correctness of distinct on tuples with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val reduceDs = ds.union(ds).distinct(_._1).map(_._1)
+
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 4 =>
+ /*
+ * check correctness of distinct on custom type with type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+
+ val reduceDs = ds.distinct(_.myInt).map( t => new Tuple1(t.myInt))
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1\n" + "2\n" + "3\n" + "4\n" + "5\n" + "6\n"
+
+ case 5 =>
+ /*
+ * check correctness of distinct on tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall3TupleDataSet(env)
+
+ val distinctDs = ds.union(ds).distinct()
+
+ distinctDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world\n"
+
+ case 6 =>
+ /*
+ * check correctness of distinct on custom type with tuple-returning type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get5TupleDataSet(env)
+
+ val reduceDs = ds.distinct( t => (t._1, t._5)).map( t => (t._1, t._5) )
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "2,1\n" + "2,2\n" + "3,2\n" + "3,3\n" + "4,1\n" + "4,2\n" + "5," +
+ "1\n" + "5,2\n" + "5,3\n"
+
+ case 7 =>
+ /*
+ * check correctness of distinct on tuples with field expressions
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getSmall5TupleDataSet(env)
+
+ val reduceDs = ds.union(ds).distinct("_1").map(t => new Tuple1(t._1))
+
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1\n" + "2\n"
+
+ case 8 =>
+ /*
+ * check correctness of distinct on Pojos
+ */
+
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getDuplicatePojoDataSet(env)
+
+ val reduceDs = ds.distinct("nestedPojo.longNumber").map(_.nestedPojo.longNumber.toInt)
+
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "10000\n20000\n30000\n"
+
+ case _ =>
+ throw new IllegalArgumentException("Invalid program id")
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class DistinctITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = DistinctProgs.runProgram(curProgId, resultPath)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object DistinctITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to DistinctProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
index b146e1c..e9d214b 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/DistinctOperatorTest.scala
@@ -17,6 +17,7 @@
*/
package org.apache.flink.api.scala.operators
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
import org.junit.Assert
import org.apache.flink.api.common.InvalidProgramException
import org.junit.Test
@@ -102,7 +103,7 @@ class DistinctOperatorTest {
}
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[RuntimeException])
def testDistinctByKeyFields2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val longDs = env.fromCollection(emptyLongData)
@@ -111,16 +112,16 @@ class DistinctOperatorTest {
longDs.distinct("_1")
}
- @Test(expected = classOf[UnsupportedOperationException])
+ @Test(expected = classOf[RuntimeException])
def testDistinctByKeyFields3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val customDs = env.fromCollection(customTypeData)
- // should not work: field key on custom type
+ // should not work: invalid fields
customDs.distinct("_1")
}
- @Test(expected = classOf[IllegalArgumentException])
+ @Test(expected = classOf[RuntimeException])
def testDistinctByKeyFields4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tupleDs = env.fromCollection(emptyTupleData)
@@ -130,11 +131,20 @@ class DistinctOperatorTest {
}
@Test
+ def testDistinctByKeyFields5(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val customDs = env.fromCollection(customTypeData)
+
+ // should work
+ customDs.distinct("myInt")
+ }
+
+ @Test
def testDistinctByKeySelector1(): Unit = {
val env: ExecutionEnvironment = ExecutionEnvironment.getExecutionEnvironment
try {
val customDs = env.fromCollection(customTypeData)
- customDs.distinct {_.l}
+ customDs.distinct {_.myLong}
}
catch {
case e: Exception => Assert.fail()
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
index d5ae6b6..f43052a 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/ExamplesITCase.scala
@@ -29,25 +29,36 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
// TODO case class Tuple2[T1, T2](_1: T1, _2: T2)
-// TODO case class Foo(a: Int, b: String)
+// TODO case class Foo(a: Int, b: String
-class Nested(var myLong: Long) {
+case class Nested(myLong: Long)
+
+class Pojo(var myString: String, var myInt: Int, var nested: Nested) {
def this() = {
- this(0);
+ this("", 0, new Nested(1))
}
+
+ def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
+
+ override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
+}
+
+class NestedPojo(var myLong: Long) {
+ def this() { this(0) }
}
-class Pojo(var myString: String, var myInt: Int, myLong: Long) {
- var nested = new Nested(myLong)
+class PojoWithPojo(var myString: String, var myInt: Int, var nested: Nested) {
def this() = {
- this("", 0, 0)
+ this("", 0, new Nested(1))
}
- override def toString() = "myString="+myString+" myInt="+myInt+" nested.myLong="+nested.myLong
+ def this(myString: String, myInt: Int, myLong: Long) { this(myString, myInt, new Nested(myLong)) }
+
+ override def toString = s"myString=$myString myInt=$myInt nested.myLong=${nested.myLong}"
}
object ExampleProgs {
- var NUM_PROGRAMS: Int = 3
+ var NUM_PROGRAMS: Int = 4
def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
progId match {
@@ -58,27 +69,53 @@ object ExampleProgs {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
- val grouped = ds.groupBy(0).reduce( { (e1, e2) => ((e1._1._1,e1._1._2), e1._2+e2._2)})
+ val grouped = ds.groupBy(0).reduce( { (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)})
grouped.writeAsText(resultPath)
env.execute()
"((this,hello),3)\n((this,is),3)\n"
+
case 2 =>
- /*
- Test nested tuples with int offset
- */
- val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
-
- val grouped = ds.groupBy("f0.f0").reduce( { (e1, e2) => ((e1._1._1,e1._1._2), e1._2+e2._2)})
- grouped.writeAsText(resultPath)
- env.execute()
- "((this,is),6)\n"
+ /*
+ Test nested tuples with int offset
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements( (("this","is"), 1), (("this", "is"),2), (("this","hello"),3) )
+
+ val grouped = ds.groupBy("_1._1").reduce{
+ (e1, e2) => ((e1._1._1, e1._1._2), e1._2 + e2._2)
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "((this,is),6)\n"
+
case 3 =>
/*
Test nested pojos
*/
val env = ExecutionEnvironment.getExecutionEnvironment
- val ds = env.fromElements( new Pojo("one", 1, 1L),new Pojo("one", 1, 1L),new Pojo("two", 666, 2L) )
+ val ds = env.fromElements(
+ new PojoWithPojo("one", 1, 1L),
+ new PojoWithPojo("one", 1, 1L),
+ new PojoWithPojo("two", 666, 2L) )
+
+ val grouped = ds.groupBy("nested.myLong").reduce {
+ (p1, p2) =>
+ p1.myInt += p2.myInt
+ p1
+ }
+ grouped.writeAsText(resultPath)
+ env.execute()
+ "myString=two myInt=666 nested.myLong=2\nmyString=one myInt=2 nested.myLong=1\n"
+
+ case 4 =>
+ /*
+ Test pojo with nested case class
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = env.fromElements(
+ new Pojo("one", 1, 1L),
+ new Pojo("one", 1, 1L),
+ new Pojo("two", 666, 2L) )
val grouped = ds.groupBy("nested.myLong").reduce {
(p1, p2) =>
@@ -124,4 +161,4 @@ object ExamplesITCase {
configs.asJavaCollection
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6be85554/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
new file mode 100644
index 0000000..b796a81
--- /dev/null
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/GroupReduceITCase.scala
@@ -0,0 +1,574 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.operators
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions._
+import org.apache.flink.api.common.operators.Order
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.ExecutionEnvironment
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets
+import org.apache.flink.api.scala.util.CollectionDataSets.CustomType
+import org.apache.flink.compiler.PactCompiler
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.test.util.JavaProgramTestBase
+import org.apache.flink.util.Collector
+import org.junit.runner.RunWith
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized
+import org.junit.runners.Parameterized.Parameters
+import org.junit.runners.Parameterized.Parameters
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.flink.api.scala._
+
+
+object GroupReduceProgs {
+ var NUM_PROGRAMS: Int = 8
+
+ def runProgram(progId: Int, resultPath: String, onCollection: Boolean): String = {
+ progId match {
+ case 1 =>
+ /*
+ * check correctness of groupReduce on tuples with key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).reduceGroup {
+ in =>
+ in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
+
+ case 2 =>
+ /*
+ * check correctness of groupReduce on tuples with multiple key field selector
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets
+ .get5TupleDataSet(env)
+ val reduceDs = ds.groupBy(4, 0).reduceGroup {
+ in =>
+ val (i, l, l2) = in
+ .map( t => (t._1, t._2, t._5))
+ .reduce((l, r) => (l._1, l._2 + r._2, l._3))
+ (i, l, 0, "P-)", l2)
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
+ "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
+ "2\n" + "5,25,0,P-),3\n"
+
+ case 3 =>
+ /*
+ * check correctness of groupReduce on tuples with key field selector and group sorting
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ env.setDegreeOfParallelism(1)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).sortGroup(2, Order.ASCENDING).reduceGroup {
+ in =>
+ in.reduce((l, r) => (l._1 + r._1, l._2, l._3 + "-" + r._3))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,Hi\n" +
+ "5,2,Hello-Hello world\n" +
+ "15,3,Hello world, how are you?-I am fine.-Luke Skywalker\n" +
+ "34,4,Comment#1-Comment#2-Comment#3-Comment#4\n" +
+ "65,5,Comment#5-Comment#6-Comment#7-Comment#8-Comment#9\n" +
+ "111,6,Comment#10-Comment#11-Comment#12-Comment#13-Comment#14-Comment#15\n"
+
+ case 4 =>
+ /*
+ * check correctness of groupReduce on tuples with key extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(_._2).reduceGroup {
+ in =>
+ in.map(t => (t._1, t._2)).reduce((l, r) => (l._1 + r._1, l._2))
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1\n" + "5,2\n" + "15,3\n" + "34,4\n" + "65,5\n" + "111,6\n"
+
+ case 5 =>
+ /*
+ * check correctness of groupReduce on custom type with type extractor
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds.groupBy(_.myInt).reduceGroup {
+ in =>
+ val iter = in.toIterator
+ val o = new CustomType
+ var c = iter.next()
+
+ o.myString = "Hello!"
+ o.myInt = c.myInt
+ o.myLong = c.myLong
+
+ while (iter.hasNext) {
+ val next = iter.next()
+ o.myLong += next.myLong
+ }
+ o
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "1,0,Hello!\n" + "2,3,Hello!\n" + "3,12,Hello!\n" + "4,30,Hello!\n" + "5,60," +
+ "Hello!\n" + "6,105,Hello!\n"
+
+ case 6 =>
+ /*
+ * check correctness of all-groupreduce for tuples
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.reduceGroup {
+ in =>
+ var i = 0
+ var l = 0L
+ for (t <- in) {
+ i += t._1
+ l += t._2
+ }
+ (i, l, "Hello World")
+ }
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "231,91,Hello World\n"
+
+ case 7 =>
+ /*
+ * check correctness of all-groupreduce for custom types
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val ds = CollectionDataSets.getCustomTypeDataSet(env)
+ val reduceDs = ds.reduceGroup {
+ in =>
+ val o = new CustomType(0, 0, "Hello!")
+ for (t <- in) {
+ o.myInt += t.myInt
+ o.myLong += t.myLong
+ }
+ o
+ }
+ reduceDs.writeAsText(resultPath)
+ env.execute()
+ "91,210,Hello!"
+
+ case 8 =>
+ /*
+ * check correctness of groupReduce with broadcast set
+ */
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val intDs = CollectionDataSets.getIntDataSet(env)
+ val ds = CollectionDataSets.get3TupleDataSet(env)
+ val reduceDs = ds.groupBy(1).reduceGroup(
+ new RichGroupReduceFunction[(Int, Long, String), (Int, Long, String)] {
+ private var f2Replace = ""
+
+ override def open(config: Configuration) {
+ val ints = this.getRuntimeContext.getBroadcastVariable[Int]("ints").asScala
+ f2Replace = ints.sum + ""
+ }
+
+ override def reduce(
+ values: Iterable[(Int, Long, String)],
+ out: Collector[(Int, Long, String)]): Unit = {
+ var i: Int = 0
+ var l: Long = 0L
+ for (t <- values.asScala) {
+ i += t._1
+ l = t._2
+ }
+ out.collect((i, l, f2Replace))
+ }
+ }).withBroadcastSet(intDs, "ints")
+ reduceDs.writeAsCsv(resultPath)
+ env.execute()
+ "1,1,55\n" + "5,2,55\n" + "15,3,55\n" + "34,4,55\n" + "65,5,55\n" + "111,6,55\n"
+
+// case 9 =>
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// val reduceDs = ds.groupBy(1).reduceGroup(new
+// GroupReduceITCase.InputReturningTuple3GroupReduce)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "11,1,Hi!\n" + "21,1,Hi again!\n" + "12,2,Hi!\n" + "22,2,Hi again!\n" + "13,2," +
+// "Hi!\n" + "23,2,Hi again!\n"
+
+// case 10 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets.getCustomTypeDataSet
+// (env)
+// val reduceDs = ds.groupBy(new
+// KeySelector[CollectionDataSets.CustomType, Integer] {
+// def getKey(in: CollectionDataSets.CustomType): Integer = {
+// return in.myInt
+// }
+// }).reduceGroup(new GroupReduceITCase.CustomTypeGroupReduceWithCombine)
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// if (collectionExecution) {
+// return null
+// }
+// else {
+// "1,0,test1\n" + "2,3,test2\n" + "3,12,test3\n" + "4,30,test4\n" + "5,60," +
+// "test5\n" + "6,105,test6\n"
+// }
+// }
+// case 11 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(2)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// val reduceDs = ds.groupBy(1).reduceGroup(new
+// GroupReduceITCase.Tuple3GroupReduceWithCombine)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// if (collectionExecution) {
+// return null
+// }
+// else {
+// "1,test1\n" + "5,test2\n" + "15,test3\n" + "34,test4\n" + "65,test5\n" + "111," +
+// "test6\n"
+// }
+// }
+//
+//
+// // all-groupreduce with combine
+//
+//
+// case 12 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// .map(new GroupReduceITCase.IdentityMapper[Tuple3[Integer, Long,
+// String]]).setParallelism(4)
+// val cfg: Configuration = new Configuration
+// cfg.setString(PactCompiler.HINT_SHIP_STRATEGY,
+// PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)
+// val reduceDs = ds.reduceGroup(new GroupReduceITCase
+// .Tuple3AllGroupReduceWithCombine).withParameters(cfg)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// if (collectionExecution) {
+// return null
+// }
+// else {
+// "322," +
+// "testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest\n"
+// }
+// }
+// case 13 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// val reduceDs = ds.groupBy(1).sortGroup(2,
+// Order.DESCENDING).reduceGroup(new GroupReduceITCase.Tuple3SortedGroupReduce)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
+// "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
+// "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
+// "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
+// }
+// case 14 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets
+// .get5TupleDataSet(env)
+// val reduceDs: DataSet[Tuple5[Integer, Long, Integer, String,
+// Long]] = ds.groupBy(new KeySelector[Tuple5[Integer, Long, Integer, String, Long],
+// Tuple2[Integer, Long]] {
+// def getKey(t: Tuple5[Integer, Long, Integer, String, Long]): Tuple2[Integer, Long] = {
+// return new Tuple2[Integer, Long](t.f0, t.f4)
+// }
+// }).reduceGroup(new GroupReduceITCase.Tuple5GroupReduce)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "1,1,0,P-),1\n" + "2,3,0,P-),1\n" + "2,2,0,P-),2\n" + "3,9,0,P-),2\n" + "3,6,0," +
+// "P-),3\n" + "4,17,0,P-),1\n" + "4,17,0,P-),2\n" + "5,11,0,P-),1\n" + "5,29,0,P-)," +
+// "2\n" + "5,25,0,P-),3\n"
+// }
+// case 15 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// val reduceDs = ds.groupBy(1).sortGroup(0,
+// Order.ASCENDING).reduceGroup(new GroupReduceITCase.OrderCheckingCombinableReduce)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "1,1,Hi\n" + "2,2,Hello\n" + "4,3,Hello world, how are you?\n" + "7,4," +
+// "Comment#1\n" + "11,5,Comment#5\n" + "16,6,Comment#10\n"
+// }
+// case 16 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets
+// .getCrazyNestedDataSet(env)
+// val reduceDs = ds.groupBy("nest_Lvl1.nest_Lvl2" +
+// ".nest_Lvl3.nest_Lvl4.f1nal").reduceGroup(new GroupReduceFunction[CollectionDataSets
+// .CrazyNested, Tuple2[String, Integer]] {
+// def reduce(values: Iterable[CollectionDataSets.CrazyNested],
+// out: Collector[Tuple2[String, Integer]]) {
+// var c: Int = 0
+// var n: String = null
+// import scala.collection.JavaConversions._
+// for (v <- values) {
+// c += 1
+// n = v.nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal
+// }
+// out.collect(new Tuple2[String, Integer](n, c))
+// }
+// })
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "aa,1\nbb,2\ncc,3\n"
+// }
+// case 17 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets
+// .getPojoExtendingFromTuple(env)
+// val reduceDs = ds.groupBy("special",
+// "f2")
+// .reduceGroup(new GroupReduceFunction[CollectionDataSets.FromTupleWithCTor, Integer] {
+// def reduce(values: Iterable[CollectionDataSets.FromTupleWithCTor],
+// out: Collector[Integer]) {
+// var c: Int = 0
+// import scala.collection.JavaConversions._
+// for (v <- values) {
+// c += 1
+// }
+// out.collect(c)
+// }
+// })
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "3\n2\n"
+// }
+// case 18 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds = CollectionDataSets
+// .getPojoContainingTupleAndWritable(env)
+// val reduceDs = ds.groupBy("hadoopFan", "theTuple.*").reduceGroup(new
+// GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, Integer] {
+// def reduce(values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
+// out: Collector[Integer]) {
+// var c: Int = 0
+// import scala.collection.JavaConversions._
+// for (v <- values) {
+// c += 1
+// }
+// out.collect(c)
+// }
+// })
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "1\n5\n"
+// }
+// case 19 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// val ds: DataSet[Tuple3[Integer, CollectionDataSets.CrazyNested,
+// CollectionDataSets.POJO]] = CollectionDataSets.getTupleContainingPojos(env)
+// val reduceDs = ds.groupBy("f0", "f1.*").reduceGroup(new
+// GroupReduceFunction[Tuple3[Integer, CollectionDataSets.CrazyNested,
+// CollectionDataSets.POJO], Integer] {
+// def reduce(values: Iterable[Tuple3[Integer, CollectionDataSets.CrazyNested,
+// CollectionDataSets.POJO]], out: Collector[Integer]) {
+// var c: Int = 0
+// import scala.collection.JavaConversions._
+// for (v <- values) {
+// c += 1
+// }
+// out.collect(c)
+// }
+// })
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "3\n1\n"
+// }
+// case 20 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets.get3TupleDataSet(env)
+// val reduceDs = ds.groupBy(1).sortGroup("f2",
+// Order.DESCENDING).reduceGroup(new GroupReduceITCase.Tuple3SortedGroupReduce)
+// reduceDs.writeAsCsv(resultPath)
+// env.execute()
+// "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello " +
+// "world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5," +
+// "Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6," +
+// "Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"
+// }
+// case 21 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets
+// .getGroupSortedNestedTupleDataSet(env)
+// val reduceDs = ds.groupBy("f1").sortGroup(0,
+// Order.DESCENDING).reduceGroup(new GroupReduceITCase.NestedTupleReducer)
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "a--(1,1)-(1,2)-(1,3)-\n" + "b--(2,2)-\n" + "c--(3,3)-(3,6)-(3,9)-\n"
+// }
+// case 22 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets
+// .getGroupSortedNestedTupleDataSet(env)
+// val reduceDs = ds.groupBy("f1").sortGroup("f0.f0",
+// Order.ASCENDING).reduceGroup(new GroupReduceITCase.NestedTupleReducer)
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "a--(1,3)-(1,2)-(2,1)-\n" + "b--(2,2)-\n" + "c--(3,3)-(3,6)-(4,9)-\n"
+// }
+// case 23 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets
+// .getGroupSortedNestedTupleDataSet(env)
+// val reduceDs = ds.groupBy("f1").sortGroup("f0.f0",
+// Order.DESCENDING).reduceGroup(new GroupReduceITCase.NestedTupleReducer)
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,3)-(3,6)-\n"
+// }
+// case 24 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets
+// .getGroupSortedNestedTupleDataSet(env)
+// val reduceDs = ds.groupBy("f1").sortGroup("f0.f0",
+// Order.DESCENDING).sortGroup("f0.f1", Order.DESCENDING).reduceGroup(new
+// GroupReduceITCase.NestedTupleReducer)
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"
+// }
+// case 25 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets
+// .getGroupSortedPojoContainingTupleAndWritable(env)
+// val reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0",
+// Order.DESCENDING)
+// .sortGroup("theTuple.f1", Order.DESCENDING)
+// .reduceGroup(new GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, String] {
+// def reduce(values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
+// out: Collector[String]) {
+// var once: Boolean = false
+// val concat: StringBuilder = new StringBuilder
+// import scala.collection.JavaConversions._
+// for (value <- values) {
+// if (!once) {
+// concat.append(value.hadoopFan.get)
+// concat.append("---")
+// once = true
+// }
+// concat.append(value.theTuple)
+// concat.append("-")
+// }
+// out.collect(concat.toString)
+// }
+// })
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "1---(10,100)-\n" + "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n"
+// }
+// case 26 => {
+// val env = ExecutionEnvironment.getExecutionEnvironment
+// env.setDegreeOfParallelism(1)
+// val ds = CollectionDataSets.getPojoWithMultiplePojos(env)
+// val reduceDs = ds.groupBy("hadoopFan")
+// .sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING)
+// .reduceGroup(new GroupReduceFunction[CollectionDataSets.PojoContainingTupleAndWritable, String] {
+// def reduce(values: Iterable[CollectionDataSets.PojoContainingTupleAndWritable],
+// out: Collector[String]) {
+// var once: Boolean = false
+// val concat: StringBuilder = new StringBuilder
+// import scala.collection.JavaConversions._
+// for (value <- values) {
+// if (!once) {
+// concat.append(value.hadoopFan.get)
+// concat.append("---")
+// once = true
+// }
+// concat.append(value.theTuple)
+// concat.append("-")
+// }
+// out.collect(concat.toString)
+// }
+// })
+// reduceDs.writeAsText(resultPath)
+// env.execute()
+// "1---(10,100)-\n" + "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n"
+// }
+// case _ => {
+// throw new IllegalArgumentException("Invalid program id")
+// }
+ }
+ }
+}
+
+
+@RunWith(classOf[Parameterized])
+class GroupReduceITCase(config: Configuration) extends JavaProgramTestBase(config) {
+
+ private var curProgId: Int = config.getInteger("ProgramId", -1)
+ private var resultPath: String = null
+ private var expectedResult: String = null
+
+ protected override def preSubmit(): Unit = {
+ resultPath = getTempDirPath("result")
+ }
+
+ protected def testProgram(): Unit = {
+ expectedResult = GroupReduceProgs.runProgram(curProgId, resultPath, isCollectionExecution)
+ }
+
+ protected override def postSubmit(): Unit = {
+ compareResultsByLinesInMemory(expectedResult, resultPath)
+ }
+}
+
+object GroupReduceITCase {
+ @Parameters
+ def getConfigurations: java.util.Collection[Array[AnyRef]] = {
+ val configs = mutable.MutableList[Array[AnyRef]]()
+ for (i <- 1 to GroupReduceProgs.NUM_PROGRAMS) {
+ val config = new Configuration()
+ config.setInteger("ProgramId", i)
+ configs += Array(config)
+ }
+
+ configs.asJavaCollection
+ }
+}
+
+
[05/11] [FLINK-1032] Rework support for POJO types in the Java API
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index eb23338..60d41d3 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -21,21 +21,26 @@ package org.apache.flink.api.java.type.extractor;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
import org.apache.flink.api.common.functions.InvalidTypesException;
import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.RichCoGroupFunction;
import org.apache.flink.api.common.functions.RichCrossFunction;
+import org.apache.flink.api.common.functions.RichFlatJoinFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichFlatJoinFunction;
-import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -43,6 +48,8 @@ import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple9;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
+import org.apache.flink.api.java.typeutils.PojoField;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -57,10 +64,14 @@ import org.apache.flink.types.Value;
import org.apache.flink.util.Collector;
import org.apache.hadoop.io.Writable;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
+import com.google.common.collect.HashMultiset;
+
public class TypeExtractorTest {
+
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testBasicType() {
@@ -140,7 +151,10 @@ public class TypeExtractorTest {
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(9, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
-
+ List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
+ ((TupleTypeInfo) ti).getKey("f3", 0, ffd);
+ Assert.assertTrue(ffd.size() == 1);
+ Assert.assertEquals(3, ffd.get(0).getPosition() );
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple9.class, tti.getTypeClass());
@@ -203,6 +217,20 @@ public class TypeExtractorTest {
Assert.assertTrue(ti.isTupleType());
Assert.assertEquals(3, ti.getArity());
Assert.assertTrue(ti instanceof TupleTypeInfo);
+ List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
+
+ ((TupleTypeInfo) ti).getKey("f0.f0", 0, ffd);
+ Assert.assertEquals(0, ffd.get(0).getPosition() );
+ ffd.clear();
+
+ ((TupleTypeInfo) ti).getKey("f0.f0", 0, ffd);
+ Assert.assertTrue( ffd.get(0).getType() instanceof BasicTypeInfo );
+ Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
+ ffd.clear();
+
+ ((TupleTypeInfo) ti).getKey("f1.f0", 0, ffd);
+ Assert.assertEquals(1, ffd.get(0).getPosition() );
+ ffd.clear();
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple3.class, tti.getTypeClass());
@@ -305,11 +333,11 @@ public class TypeExtractorTest {
Assert.assertFalse(ti.isBasicType());
Assert.assertFalse(ti.isTupleType());
- Assert.assertTrue(ti instanceof GenericTypeInfo);
+ Assert.assertTrue(ti instanceof PojoTypeInfo);
Assert.assertEquals(ti.getTypeClass(), CustomType.class);
// use getForClass()
- Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof GenericTypeInfo);
+ Assert.assertTrue(TypeExtractor.getForClass(CustomType.class) instanceof PojoTypeInfo);
Assert.assertEquals(TypeExtractor.getForClass(CustomType.class).getTypeClass(), ti.getTypeClass());
// use getForObject()
@@ -318,10 +346,454 @@ public class TypeExtractorTest {
Assert.assertFalse(ti2.isBasicType());
Assert.assertFalse(ti2.isTupleType());
- Assert.assertTrue(ti2 instanceof GenericTypeInfo);
+ Assert.assertTrue(ti2 instanceof PojoTypeInfo);
Assert.assertEquals(ti2.getTypeClass(), CustomType.class);
}
+
+ //
+ // Pojo Type tests
+ // A Pojo is a bean-style class with getters, setters and empty ctor
+ // OR a class with all fields public (or for every private field, there has to be a public getter/setter)
+ // everything else is a generic type (that can't be used for field selection)
+ //
+
+
+ // test with correct pojo types
+ public static class WC { // is a pojo
+ public ComplexNestedClass complex; // is a pojo
+ private int count; // is a BasicType
+ public WC() {
+ }
+ public int getCount() {
+ return count;
+ }
+ public void setCount(int c) {
+ this.count = c;
+ }
+ }
+ public static class ComplexNestedClass { // pojo
+ public static int ignoreStaticField;
+ public transient int ignoreTransientField;
+ public Date date; // generic type
+ public Integer someNumber; // BasicType
+ public float someFloat; // BasicType
+ public Tuple3<Long, Long, String> word; //Tuple Type with three basic types
+ public Object nothing; // generic type
+ public MyWritable hadoopCitizen; // writableType
+ }
+
+ // all public test
+ public static class AllPublic extends ComplexNestedClass {
+ public ArrayList<String> somethingFancy; // generic type
+ public HashMultiset<Integer> fancyIds; // generic type
+ public String[] fancyArray; // generic type
+ }
+
+ public static class ParentSettingGenerics extends PojoWithGenerics<Integer, Long> {
+ public String field3;
+ }
+ public static class PojoWithGenerics<T1, T2> {
+ public int key;
+ public T1 field1;
+ public T2 field2;
+ }
+
+ public static class ComplexHierarchyTop extends ComplexHierarchy<Tuple1<String>> {}
+ public static class ComplexHierarchy<T> extends PojoWithGenerics<FromTuple,T> {}
+
+ // extends from Tuple and adds a field
+ public static class FromTuple extends Tuple3<String, String, Long> {
+ private static final long serialVersionUID = 1L;
+ public int special;
+ }
+
+ public static class IncorrectPojo {
+ private int isPrivate;
+ public int getIsPrivate() {
+ return isPrivate;
+ }
+ // setter is missing (intentional)
+ }
+
+ // correct pojo
+ public static class BeanStylePojo {
+ public String abc;
+ private int field;
+ public int getField() {
+ return this.field;
+ }
+ public void setField(int f) {
+ this.field = f;
+ }
+ }
+ public static class WrongCtorPojo {
+ public int a;
+ public WrongCtorPojo(int a) {
+ this.a = a;
+ }
+ }
+
+ // in this test, the location of the getters and setters is mixed across the type hierarchy.
+ public static class TypedPojoGetterSetterCheck extends GenericPojoGetterSetterCheck<String> {
+ public void setPackageProtected(String in) {
+ this.packageProtected = in;
+ }
+ }
+ public static class GenericPojoGetterSetterCheck<T> {
+ T packageProtected;
+ public T getPackageProtected() {
+ return packageProtected;
+ }
+ }
+
+ @Test
+ public void testIncorrectPojos() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(IncorrectPojo.class);
+ Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
+
+ typeForClass = TypeExtractor.createTypeInfo(WrongCtorPojo.class);
+ Assert.assertTrue(typeForClass instanceof GenericTypeInfo<?>);
+ }
+
+ @Test
+ public void testCorrectPojos() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(BeanStylePojo.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+
+ typeForClass = TypeExtractor.createTypeInfo(TypedPojoGetterSetterCheck.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ }
+
+ @Test
+ public void testPojoWC() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(WC.class);
+ checkWCPojoAsserts(typeForClass);
+
+ WC t = new WC();
+ t.complex = new ComplexNestedClass();
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(t);
+ checkWCPojoAsserts(typeForObject);
+ }
+
+ private void checkWCPojoAsserts(TypeInformation<?> typeInfo) {
+ Assert.assertFalse(typeInfo.isBasicType());
+ Assert.assertFalse(typeInfo.isTupleType());
+ Assert.assertEquals(9, typeInfo.getTotalFields());
+ Assert.assertTrue(typeInfo instanceof PojoTypeInfo);
+ PojoTypeInfo<?> pojoType = (PojoTypeInfo<?>) typeInfo;
+
+ List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
+ String[] fields = {"count","complex.date", "complex.hadoopCitizen", "complex.nothing",
+ "complex.someFloat", "complex.someNumber", "complex.word.f0",
+ "complex.word.f1", "complex.word.f2"};
+ int[] positions = {8,0,1,2,
+ 3,4,5,
+ 6,7};
+ Assert.assertEquals(fields.length, positions.length);
+ for(int i = 0; i < fields.length; i++) {
+ pojoType.getKey(fields[i], 0, ffd);
+ Assert.assertEquals("Too many keys returned", 1, ffd.size());
+ Assert.assertEquals("position of field "+fields[i]+" wrong", positions[i], ffd.get(0).getPosition());
+ ffd.clear();
+ }
+
+ pojoType.getKey("complex.word.*", 0, ffd);
+ Assert.assertEquals(3, ffd.size());
+ // check if it returns 5,6,7
+ for(FlatFieldDescriptor ffdE : ffd) {
+ final int pos = ffdE.getPosition();
+ Assert.assertTrue(pos <= 7 );
+ Assert.assertTrue(5 <= pos );
+ if(pos == 5) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 6) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 7) {
+ Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+
+ pojoType.getKey("complex.*", 0, ffd);
+ Assert.assertEquals(8, ffd.size());
+ // check if it returns 0-7
+ for(FlatFieldDescriptor ffdE : ffd) {
+ final int pos = ffdE.getPosition();
+ Assert.assertTrue(ffdE.getPosition() <= 7 );
+ Assert.assertTrue(0 <= ffdE.getPosition() );
+ if(pos == 0) {
+ Assert.assertEquals(Date.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 1) {
+ Assert.assertEquals(MyWritable.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 2) {
+ Assert.assertEquals(Object.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 3) {
+ Assert.assertEquals(Float.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 4) {
+ Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 5) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 6) {
+ Assert.assertEquals(Long.class, ffdE.getType().getTypeClass());
+ }
+ if(pos == 7) {
+ Assert.assertEquals(String.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+ pojoType.getKey("*", 0, ffd);
+ Assert.assertEquals(9, ffd.size());
+ // check if it returns 0-8
+ for(FlatFieldDescriptor ffdE : ffd) {
+ Assert.assertTrue(ffdE.getPosition() <= 8 );
+ Assert.assertTrue(0 <= ffdE.getPosition() );
+ if(ffdE.getPosition() == 8) {
+ Assert.assertEquals(Integer.class, ffdE.getType().getTypeClass());
+ }
+ }
+ ffd.clear();
+
+ TypeInformation<?> typeComplexNested = pojoType.getTypeAt(0); // ComplexNestedClass complex
+ Assert.assertTrue(typeComplexNested instanceof PojoTypeInfo);
+
+ Assert.assertEquals(6, typeComplexNested.getArity());
+ Assert.assertEquals(8, typeComplexNested.getTotalFields());
+ PojoTypeInfo<?> pojoTypeComplexNested = (PojoTypeInfo<?>) typeComplexNested;
+
+ boolean dateSeen = false, intSeen = false, floatSeen = false,
+ tupleSeen = false, objectSeen = false, writableSeen = false;
+ for(int i = 0; i < pojoTypeComplexNested.getArity(); i++) {
+ PojoField field = pojoTypeComplexNested.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("date")) {
+ if(dateSeen) {
+ Assert.fail("already seen");
+ }
+ dateSeen = true;
+ Assert.assertEquals(new GenericTypeInfo<Date>(Date.class), field.type);
+ Assert.assertEquals(Date.class, field.type.getTypeClass());
+ } else if(name.equals("someNumber")) {
+ if(intSeen) {
+ Assert.fail("already seen");
+ }
+ intSeen = true;
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ Assert.assertEquals(Integer.class, field.type.getTypeClass());
+ } else if(name.equals("someFloat")) {
+ if(floatSeen) {
+ Assert.fail("already seen");
+ }
+ floatSeen = true;
+ Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, field.type);
+ Assert.assertEquals(Float.class, field.type.getTypeClass());
+ } else if(name.equals("word")) {
+ if(tupleSeen) {
+ Assert.fail("already seen");
+ }
+ tupleSeen = true;
+ Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
+ Assert.assertEquals(Tuple3.class, field.type.getTypeClass());
+ // do some more advanced checks on the tuple
+ TupleTypeInfo<?> tupleTypeFromComplexNested = (TupleTypeInfo<?>) field.type;
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(0));
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(1));
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, tupleTypeFromComplexNested.getTypeAt(2));
+ } else if(name.equals("nothing")) {
+ if(objectSeen) {
+ Assert.fail("already seen");
+ }
+ objectSeen = true;
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ Assert.assertEquals(Object.class, field.type.getTypeClass());
+ } else if(name.equals("hadoopCitizen")) {
+ if(writableSeen) {
+ Assert.fail("already seen");
+ }
+ writableSeen = true;
+ Assert.assertEquals(new WritableTypeInfo<MyWritable>(MyWritable.class), field.type);
+ Assert.assertEquals(MyWritable.class, field.type.getTypeClass());
+ } else {
+ Assert.fail("field "+field+" is not expected");
+ }
+ }
+ Assert.assertTrue("Field was not present", dateSeen);
+ Assert.assertTrue("Field was not present", intSeen);
+ Assert.assertTrue("Field was not present", floatSeen);
+ Assert.assertTrue("Field was not present", tupleSeen);
+ Assert.assertTrue("Field was not present", objectSeen);
+ Assert.assertTrue("Field was not present", writableSeen);
+
+ TypeInformation<?> typeAtOne = pojoType.getTypeAt(1); // int count
+ Assert.assertTrue(typeAtOne instanceof BasicTypeInfo);
+
+ Assert.assertEquals(typeInfo.getTypeClass(), WC.class);
+ Assert.assertEquals(typeInfo.getArity(), 2);
+ }
+
+ @Test
+ public void testPojoAllPublic() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(AllPublic.class);
+ checkAllPublicAsserts(typeForClass);
+
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(new AllPublic() );
+ checkAllPublicAsserts(typeForObject);
+ }
+
+ private void checkAllPublicAsserts(TypeInformation<?> typeInformation) {
+ Assert.assertTrue(typeInformation instanceof PojoTypeInfo);
+ Assert.assertEquals(9, typeInformation.getArity());
+ Assert.assertEquals(11, typeInformation.getTotalFields());
+ // check if the three additional fields are identified correctly
+ boolean arrayListSeen = false, multisetSeen = false, strArraySeen = false;
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("somethingFancy")) {
+ if(arrayListSeen) {
+ Assert.fail("already seen");
+ }
+ arrayListSeen = true;
+ Assert.assertTrue(field.type instanceof GenericTypeInfo);
+ Assert.assertEquals(ArrayList.class, field.type.getTypeClass());
+ } else if(name.equals("fancyIds")) {
+ if(multisetSeen) {
+ Assert.fail("already seen");
+ }
+ multisetSeen = true;
+ Assert.assertTrue(field.type instanceof GenericTypeInfo);
+ Assert.assertEquals(HashMultiset.class, field.type.getTypeClass());
+ } else if(name.equals("fancyArray")) {
+ if(strArraySeen) {
+ Assert.fail("already seen");
+ }
+ strArraySeen = true;
+ Assert.assertEquals(BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, field.type);
+ Assert.assertEquals(String[].class, field.type.getTypeClass());
+ } else if(Arrays.asList("date", "someNumber", "someFloat", "word", "nothing", "hadoopCitizen").contains(name)) {
+ // ignore these, they are inherited from the ComplexNestedClass
+ }
+ else {
+ Assert.fail("field "+field+" is not expected");
+ }
+ }
+ Assert.assertTrue("Field was not present", arrayListSeen);
+ Assert.assertTrue("Field was not present", multisetSeen);
+ Assert.assertTrue("Field was not present", strArraySeen);
+ }
+
+ @Test
+ public void testPojoExtendingTuple() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(FromTuple.class);
+ checkFromTuplePojo(typeForClass);
+
+ FromTuple ft = new FromTuple();
+ ft.f0 = ""; ft.f1 = ""; ft.f2 = 0L;
+ TypeInformation<?> typeForObject = TypeExtractor.getForObject(ft);
+ checkFromTuplePojo(typeForObject);
+ }
+
+ private void checkFromTuplePojo(TypeInformation<?> typeInformation) {
+ Assert.assertTrue(typeInformation instanceof PojoTypeInfo<?>);
+ Assert.assertEquals(4, typeInformation.getTotalFields());
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("special")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else if(name.equals("f0") || name.equals("f1")) {
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
+ } else if(name.equals("f2")) {
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("unexpected field");
+ }
+ }
+ }
+
+ @Test
+ public void testPojoWithGenerics() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ParentSettingGenerics.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else if (name.equals("field2")) {
+ Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.type);
+ } else if (name.equals("field3")) {
+ Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.type);
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+ /**
+ * Test if the TypeExtractor is accepting untyped generics,
+ * making them GenericTypes
+ */
+ @Test
+ @Ignore // kryo needed.
+ public void testPojoWithGenericsSomeFieldsGeneric() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(PojoWithGenerics.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ } else if (name.equals("field2")) {
+ Assert.assertEquals(new GenericTypeInfo<Object>(Object.class), field.type);
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+
+ @Test
+ public void testPojoWithComplexHierarchy() {
+ TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ComplexHierarchyTop.class);
+ Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>);
+ PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass;
+ for(int i = 0; i < pojoTypeForClass.getArity(); i++) {
+ PojoField field = pojoTypeForClass.getPojoFieldAt(i);
+ String name = field.field.getName();
+ if(name.equals("field1")) {
+ Assert.assertTrue(field.type instanceof PojoTypeInfo<?>); // From tuple is pojo (not tuple type!)
+ } else if (name.equals("field2")) {
+ Assert.assertTrue(field.type instanceof TupleTypeInfo<?>);
+ Assert.assertTrue( ((TupleTypeInfo<?>)field.type).getTypeAt(0).equals(BasicTypeInfo.STRING_TYPE_INFO) );
+ } else if (name.equals("key")) {
+ Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.type);
+ } else {
+ Assert.fail("Unexpected field "+field);
+ }
+ }
+ }
+
+ // End of Pojo type tests
+
public static class CustomType {
public String myField1;
public int myField2;
@@ -356,9 +828,27 @@ public class TypeExtractorTest {
TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
Assert.assertEquals(Tuple2.class, tti.getTypeClass());
+ List<FlatFieldDescriptor> ffd = new ArrayList<FlatFieldDescriptor>();
+
+ tti.getKey("f0", 0, ffd);
+ Assert.assertEquals(1, ffd.size());
+ Assert.assertEquals(0, ffd.get(0).getPosition() ); // Long
+ Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(Long.class) );
+ ffd.clear();
+
+ tti.getKey("f1.myField1", 0, ffd);
+ Assert.assertEquals(1, ffd.get(0).getPosition() );
+ Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(String.class) );
+ ffd.clear();
+
+
+ tti.getKey("f1.myField2", 0, ffd);
+ Assert.assertEquals(2, ffd.get(0).getPosition() );
+ Assert.assertTrue( ffd.get(0).getType().getTypeClass().equals(Integer.class) );
+
Assert.assertEquals(Long.class, tti.getTypeAt(0).getTypeClass());
- Assert.assertTrue(tti.getTypeAt(1) instanceof GenericTypeInfo);
+ Assert.assertTrue(tti.getTypeAt(1) instanceof PojoTypeInfo);
Assert.assertEquals(CustomType.class, tti.getTypeAt(1).getTypeClass());
// use getForObject()
@@ -371,7 +861,7 @@ public class TypeExtractorTest {
Assert.assertEquals(Tuple2.class, tti2.getTypeClass());
Assert.assertEquals(Long.class, tti2.getTypeAt(0).getTypeClass());
- Assert.assertTrue(tti2.getTypeAt(1) instanceof GenericTypeInfo);
+ Assert.assertTrue(tti2.getTypeAt(1) instanceof PojoTypeInfo);
Assert.assertEquals(CustomType.class, tti2.getTypeAt(1).getTypeClass());
}
@@ -1203,6 +1693,7 @@ public class TypeExtractorTest {
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
+ @Ignore
public void testParamertizedCustomObject() {
RichMapFunction<?, ?> function = new RichMapFunction<MyObject<String>, MyObject<String>>() {
private static final long serialVersionUID = 1L;
@@ -1214,7 +1705,7 @@ public class TypeExtractorTest {
};
TypeInformation<?> ti = TypeExtractor.getMapReturnTypes(function, (TypeInformation) TypeInfoParser.parse("org.apache.flink.api.java.type.extractor.TypeExtractorTest$MyObject"));
- Assert.assertTrue(ti instanceof GenericTypeInfo);
+ Assert.assertTrue(ti instanceof PojoTypeInfo);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/GenericPairComparatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/GenericPairComparatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/GenericPairComparatorTest.java
index 382b009..06330d3 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/GenericPairComparatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/GenericPairComparatorTest.java
@@ -28,7 +28,6 @@ import org.apache.flink.api.common.typeutils.base.IntSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple4;
-
import org.apache.flink.api.java.typeutils.runtime.tuple.base.TuplePairComparatorTestBase;
public class GenericPairComparatorTest extends TuplePairComparatorTestBase<Tuple3<Integer, String, Double>, Tuple4<Integer, Float, Long, Double>> {
@@ -57,7 +56,7 @@ public class GenericPairComparatorTest extends TuplePairComparatorTestBase<Tuple
new Tuple4<Integer, Float, Long, Double>(7, 0.88f, 34L, 15.2)
};
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
@Override
protected GenericPairComparator<Tuple3<Integer, String, Double>, Tuple4<Integer, Float, Long, Double>> createComparator(boolean ascending) {
int[] fields1 = new int[]{0, 2};
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
index fcc84f0..5d0917e 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializerTest.java
@@ -18,27 +18,37 @@
package org.apache.flink.api.java.typeutils.runtime;
-import com.google.common.base.Objects;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.SerializerTestBase;
+import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.junit.Ignore;
+import org.junit.Assert;
+import org.junit.Test;
-import java.util.Random;
+import com.google.common.base.Objects;
/**
* A test for the {@link org.apache.flink.api.java.typeutils.runtime.PojoSerializer}.
*/
-@Ignore
public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.TestUserClass> {
private TypeInformation<TestUserClass> type = TypeExtractor.getForClass(TestUserClass.class);
@Override
protected TypeSerializer<TestUserClass> createSerializer() {
TypeSerializer<TestUserClass> serializer = type.createSerializer();
- assert (serializer instanceof PojoSerializer);
+ assert(serializer instanceof PojoSerializer);
return serializer;
}
@@ -67,12 +77,12 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
// User code class for testing the serializer
public static class TestUserClass {
- private int dumm1;
- protected String dumm2;
+ public int dumm1;
+ public String dumm2;
public double dumm3;
- private int[] dumm4;
+ public int[] dumm4;
- private NestedTestUserClass nestedClass;
+ public NestedTestUserClass nestedClass;
public TestUserClass() {
}
@@ -121,10 +131,10 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
}
public static class NestedTestUserClass {
- private int dumm1;
- protected String dumm2;
+ public int dumm1;
+ public String dumm2;
public double dumm3;
- private int[] dumm4;
+ public int[] dumm4;
public NestedTestUserClass() {
}
@@ -167,4 +177,52 @@ public class PojoSerializerTest extends SerializerTestBase<PojoSerializerTest.Te
return true;
}
}
+
+ /**
+ * This tests if the hashes returned by the pojo and tuple comparators are the same
+ */
+ @Test
+ public void testTuplePojoTestEquality() {
+
+ // test with a simple, string-key first.
+ PojoTypeInfo<TestUserClass> pType = (PojoTypeInfo<TestUserClass>) type;
+ List<FlatFieldDescriptor> result = new ArrayList<FlatFieldDescriptor>();
+ pType.getKey("nestedClass.dumm2", 0, result);
+ int[] fields = new int[1]; // see below
+ fields[0] = result.get(0).getPosition();
+ TypeComparator<TestUserClass> pojoComp = pType.createComparator( fields, new boolean[]{true}, 0);
+
+ TestUserClass pojoTestRecord = new TestUserClass(0, "abc", 3d, new int[] {1,2,3}, new NestedTestUserClass(1, "haha", 4d, new int[] {5,4,3}));
+ int pHash = pojoComp.hash(pojoTestRecord);
+
+ Tuple1<String> tupleTest = new Tuple1<String>("haha");
+ TupleTypeInfo<Tuple1<String>> tType = (TupleTypeInfo<Tuple1<String>>)TypeExtractor.getForObject(tupleTest);
+ TypeComparator<Tuple1<String>> tupleComp = tType.createComparator(new int[] {0}, new boolean[] {true}, 0);
+
+ int tHash = tupleComp.hash(tupleTest);
+
+ Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable", pHash == tHash);
+
+ Tuple3<Integer, String, Double> multiTupleTest = new Tuple3<Integer, String, Double>(1, "haha", 4d); // its important here to use the same values.
+ TupleTypeInfo<Tuple3<Integer, String, Double>> multiTupleType = (TupleTypeInfo<Tuple3<Integer, String, Double>>)TypeExtractor.getForObject(multiTupleTest);
+
+ ExpressionKeys fieldKey = new ExpressionKeys(new int[]{1,0,2}, multiTupleType);
+ ExpressionKeys expressKey = new ExpressionKeys(new String[] {"nestedClass.dumm2", "nestedClass.dumm1", "nestedClass.dumm3"}, pType);
+ try {
+ Assert.assertTrue("Expecting the keys to be compatible", fieldKey.areCompatible(expressKey));
+ } catch (IncompatibleKeysException e) {
+ e.printStackTrace();
+ Assert.fail("Keys must be compatible: "+e.getMessage());
+ }
+ TypeComparator<TestUserClass> multiPojoComp = pType.createComparator( expressKey.computeLogicalKeyPositions(), new boolean[]{true, true, true}, 0);
+ int multiPojoHash = multiPojoComp.hash(pojoTestRecord);
+
+
+ // pojo order is: dumm2 (str), dumm1 (int), dumm3 (double).
+ TypeComparator<Tuple3<Integer, String, Double>> multiTupleComp = multiTupleType.createComparator(fieldKey.computeLogicalKeyPositions(), new boolean[] {true, true,true}, 0);
+ int multiTupleHash = multiTupleComp.hash(multiTupleTest);
+
+ Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable. Also for those with multiple key fields", multiPojoHash == multiTupleHash);
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java
index 1cbc1d0..ce442d1 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/TupleSerializerTest.java
@@ -22,6 +22,7 @@ package org.apache.flink.api.java.typeutils.runtime;
import java.util.ArrayList;
import java.util.Random;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -208,7 +209,7 @@ public class TupleSerializerTest {
private final <T extends Tuple> void runTests(T... instances) {
try {
TupleTypeInfo<T> tupleTypeInfo = (TupleTypeInfo<T>) TypeExtractor.getForObject(instances[0]);
- TupleSerializer<T> serializer = tupleTypeInfo.createSerializer();
+ TypeSerializer<T> serializer = tupleTypeInfo.createSerializer();
Class<T> tupleClass = tupleTypeInfo.getTypeClass();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
index ecc5259..d249e9a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/GroupReduceDriverTest.java
@@ -50,7 +50,7 @@ public class GroupReduceDriverTest {
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -82,7 +82,7 @@ public class GroupReduceDriverTest {
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -118,7 +118,7 @@ public class GroupReduceDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
@@ -154,7 +154,7 @@ public class GroupReduceDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
@@ -196,7 +196,7 @@ public class GroupReduceDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
index e189790..44cbe16 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceCombineDriverTest.java
@@ -55,7 +55,7 @@ public class ReduceCombineDriverTest {
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -90,7 +90,7 @@ public class ReduceCombineDriverTest {
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -121,7 +121,7 @@ public class ReduceCombineDriverTest {
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -160,7 +160,7 @@ public class ReduceCombineDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
@@ -188,7 +188,7 @@ public class ReduceCombineDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
index d7c14fb..ae4e54c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/ReduceDriverTest.java
@@ -50,7 +50,7 @@ public class ReduceDriverTest {
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
context.setDriverStrategy(DriverStrategy.SORTED_REDUCE);
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -82,7 +82,7 @@ public class ReduceDriverTest {
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -110,7 +110,7 @@ public class ReduceDriverTest {
List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer());
@@ -148,7 +148,7 @@ public class ReduceDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
@@ -175,7 +175,7 @@ public class ReduceDriverTest {
List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer());
- TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true});
+ TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0);
GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/MassiveStringSortingITCase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/MassiveStringSortingITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/MassiveStringSortingITCase.java
index a955302..257eb87 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/MassiveStringSortingITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/sort/MassiveStringSortingITCase.java
@@ -180,7 +180,7 @@ public class MassiveStringSortingITCase {
TupleTypeInfo<Tuple2<String, String[]>> typeInfo = (TupleTypeInfo<Tuple2<String, String[]>>) (TupleTypeInfo<?>) TypeInfoParser.parse("Tuple2<String, String[]>");
TypeSerializer<Tuple2<String, String[]>> serializer = typeInfo.createSerializer();
- TypeComparator<Tuple2<String, String[]>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true } );
+ TypeComparator<Tuple2<String, String[]>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0);
reader = new BufferedReader(new FileReader(input));
MutableObjectIterator<Tuple2<String, String[]>> inputIterator = new StringTupleReaderMutableObjectIterator(reader);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListComparator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListComparator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListComparator.java
index 49d31a2..b5822a8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListComparator.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntListComparator.java
@@ -33,8 +33,6 @@ public class IntListComparator extends TypeComparator<IntList> {
private int reference;
- private Comparable[] extractedKey = new Comparable[1];
-
private final TypeComparator[] comparators = new TypeComparator[] {new IntComparator(true)};
@Override
@@ -142,12 +140,12 @@ public class IntListComparator extends TypeComparator<IntList> {
}
@Override
- public Object[] extractKeys(IntList record) {
- extractedKey[0] = record.getKey();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = (Comparable) record;
+ return 1;
}
- @Override public TypeComparator[] getComparators() {
+ @Override public TypeComparator[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairComparator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairComparator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairComparator.java
index 98c5b69..ab20b7f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairComparator.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/IntPairComparator.java
@@ -33,8 +33,6 @@ public class IntPairComparator extends TypeComparator<IntPair> {
private int reference;
- private final Comparable[] extractedKey = new Comparable[1];
-
private final TypeComparator[] comparators = new TypeComparator[] {new IntComparator(true)};
@Override
@@ -117,11 +115,12 @@ public class IntPairComparator extends TypeComparator<IntPair> {
}
@Override
- public Object[] extractKeys(IntPair pair) {
- extractedKey[0] = pair.getKey();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = ((IntPair) record).getKey();
+ return 1;
}
- @Override public TypeComparator[] getComparators() {
+
+ @Override public TypeComparator[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairComparator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairComparator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairComparator.java
index a5355dd..0b0446f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairComparator.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/types/StringPairComparator.java
@@ -34,8 +34,6 @@ public class StringPairComparator extends TypeComparator<StringPair> {
private String reference;
- private Comparable[] extractedKey = new Comparable[1];
-
private final TypeComparator[] comparators = new TypeComparator[] {new StringComparator(true)};
@Override
@@ -118,12 +116,12 @@ public class StringPairComparator extends TypeComparator<StringPair> {
}
@Override
- public Object[] extractKeys(StringPair record) {
- extractedKey[0] = record.getKey();
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = ((StringPair) record).getKey();
+ return 1;
}
- @Override public TypeComparator[] getComparators() {
+ @Override public TypeComparator[] getFlatComparators() {
return comparators;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
index 765d121..a717f55 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/util/OutputEmitterTest.java
@@ -382,8 +382,6 @@ public class OutputEmitterTest extends TestCase {
@SuppressWarnings({"serial", "rawtypes"})
private static class TestIntComparator extends TypeComparator<Integer> {
- private final Comparable[] extractedKey = new Comparable[1];
-
private TypeComparator[] comparators = new TypeComparator[]{new IntComparator(true)};
@Override
@@ -444,13 +442,13 @@ public class OutputEmitterTest extends TestCase {
public TypeComparator<Integer> duplicate() { throw new UnsupportedOperationException(); }
@Override
- public Object[] extractKeys(Integer record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
@Override
- public TypeComparator[] getComparators() {
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/java/org/apache/flink/api/scala/operators/ScalaAggregateOperator.java
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/operators/ScalaAggregateOperator.java b/flink-scala/src/main/java/org/apache/flink/api/scala/operators/ScalaAggregateOperator.java
index 310fc17..534ef45 100644
--- a/flink-scala/src/main/java/org/apache/flink/api/scala/operators/ScalaAggregateOperator.java
+++ b/flink-scala/src/main/java/org/apache/flink/api/scala/operators/ScalaAggregateOperator.java
@@ -198,7 +198,7 @@ public class ScalaAggregateOperator<IN> extends SingleInputOperator<IN, IN, Scal
return po;
}
- if (this.grouping.getKeys() instanceof Keys.FieldPositionKeys) {
+ if (this.grouping.getKeys() instanceof Keys.ExpressionKeys) {
// grouped aggregation
int[] logicalKeyPositions = this.grouping.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, IN> operatorInfo = new UnaryOperatorInformation<IN, IN>(getInputType(), getResultType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
index 8b452ae..28624bc 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
@@ -27,7 +27,7 @@ import org.apache.flink.api.java.aggregation.Aggregations
import org.apache.flink.api.java.functions.{FirstReducer, KeySelector}
import org.apache.flink.api.java.io.{PrintingOutputFormat, TextOutputFormat}
import org.apache.flink.api.java.operators.JoinOperator.JoinHint
-import org.apache.flink.api.java.operators.Keys.FieldPositionKeys
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys
import org.apache.flink.api.java.operators._
import org.apache.flink.api.java.{DataSet => JavaDataSet}
import org.apache.flink.api.scala.operators.{ScalaCsvOutputFormat, ScalaAggregateOperator}
@@ -544,7 +544,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
def distinct(fields: Int*): DataSet[T] = {
wrap(new DistinctOperator[T](
javaSet,
- new Keys.FieldPositionKeys[T](fields.toArray, javaSet.getType, true)))
+ new Keys.ExpressionKeys[T](fields.toArray, javaSet.getType, true)))
}
/**
@@ -557,7 +557,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
val fieldIndices = fieldNames2Indices(javaSet.getType, firstField +: otherFields.toArray)
wrap(new DistinctOperator[T](
javaSet,
- new Keys.FieldPositionKeys[T](fieldIndices, javaSet.getType, true)))
+ new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType, true)))
}
/**
@@ -602,7 +602,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
def groupBy(fields: Int*): GroupedDataSet[T] = {
new GroupedDataSet[T](
this,
- new Keys.FieldPositionKeys[T](fields.toArray, javaSet.getType,false))
+ new Keys.ExpressionKeys[T](fields.toArray, javaSet.getType,false))
}
/**
@@ -619,7 +619,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
new GroupedDataSet[T](
this,
- new Keys.FieldPositionKeys[T](fieldIndices, javaSet.getType,false))
+ new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType,false))
}
// public UnsortedGrouping<T> groupBy(String... fields) {
@@ -835,7 +835,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
*/
def iterateDelta[R: ClassTag](workset: DataSet[R], maxIterations: Int, keyFields: Array[Int])(
stepFunction: (DataSet[T], DataSet[R]) => (DataSet[T], DataSet[R])) = {
- val key = new FieldPositionKeys[T](keyFields, javaSet.getType, false)
+ val key = new ExpressionKeys[T](keyFields, javaSet.getType, false)
val iterativeSet = new DeltaIteration[T, R](
javaSet.getExecutionEnvironment,
@@ -864,8 +864,8 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
stepFunction: (DataSet[T], DataSet[R]) => (DataSet[T], DataSet[R])) = {
val fieldIndices = fieldNames2Indices(javaSet.getType, keyFields)
- val key = new FieldPositionKeys[T](fieldIndices, javaSet.getType, false)
+ val key = new ExpressionKeys[T](fieldIndices, javaSet.getType, false)
val iterativeSet = new DeltaIteration[T, R](
javaSet.getExecutionEnvironment,
javaSet.getType,
@@ -920,7 +920,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
val op = new PartitionOperator[T](
javaSet,
PartitionMethod.HASH,
- new Keys.FieldPositionKeys[T](fields.toArray, javaSet.getType, false))
+ new Keys.ExpressionKeys[T](fields.toArray, javaSet.getType, false))
wrap(op)
}
@@ -936,7 +936,7 @@ class DataSet[T: ClassTag](set: JavaDataSet[T]) {
val op = new PartitionOperator[T](
javaSet,
PartitionMethod.HASH,
- new Keys.FieldPositionKeys[T](fieldIndices, javaSet.getType, false))
+ new Keys.ExpressionKeys[T](fieldIndices, javaSet.getType, false))
wrap(op)
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/scala/org/apache/flink/api/scala/crossDataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/crossDataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/crossDataSet.scala
index fd1985d..f67f10c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/crossDataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/crossDataSet.scala
@@ -19,7 +19,7 @@ package org.apache.flink.api.scala
import org.apache.commons.lang3.Validate
import org.apache.flink.api.common.functions.{RichCrossFunction, CrossFunction}
-import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
import org.apache.flink.api.java.operators._
import org.apache.flink.api.java.{DataSet => JavaDataSet}
import org.apache.flink.api.scala.typeutils.{CaseClassSerializer, CaseClassTypeInfo}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassComparator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassComparator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassComparator.scala
index 83fe5cf..1353b44 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassComparator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassComparator.scala
@@ -139,10 +139,10 @@ class CaseClassComparator[T <: Product](
}
}
- def extractKeys(value: T) = {
+ def extractKeys(value: AnyRef, target: Array[AnyRef], index: Int) = {
for (i <- 0 until keyPositions.length ) {
- extractedKeys(i) = value.productElement(keyPositions(i)).asInstanceOf[AnyRef]
+ target(index + i) = value.asInstanceOf[T].productElement(keyPositions(i)).asInstanceOf[AnyRef]
}
- extractedKeys
+ keyPositions.length
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
index 153f3a8..c9a3bbf 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
@@ -32,7 +32,8 @@ abstract class CaseClassTypeInfo[T <: Product](
val fieldNames: Seq[String])
extends TupleTypeInfoBase[T](clazz, fieldTypes: _*) {
- def createComparator(logicalKeyFields: Array[Int], orders: Array[Boolean]): TypeComparator[T] = {
+ override def createComparator(logicalKeyFields: Array[Int],
+ orders: Array[Boolean], offset: Int): TypeComparator[T] = {
// sanity checks
if (logicalKeyFields == null || orders == null
|| logicalKeyFields.length != orders.length || logicalKeyFields.length > types.length) {
@@ -80,6 +81,21 @@ abstract class CaseClassTypeInfo[T <: Product](
fields map { x => fieldNames.indexOf(x) }
}
+ override protected def initializeNewComparator(localKeyCount: Int): Unit = {
+ throw new UnsupportedOperationException("The Scala API is not using the composite " +
+ "type comparator creation")
+ }
+
+ override protected def getNewComparator: TypeComparator[T] = {
+ throw new UnsupportedOperationException("The Scala API is not using the composite " +
+ "type comparator creation")
+ }
+
+ override protected def addCompareField(fieldId: Int, comparator: TypeComparator[_]): Unit = {
+ throw new UnsupportedOperationException("The Scala API is not using the composite " +
+ "type comparator creation")
+ }
+
override def toString = clazz.getSimpleName + "(" + fieldNames.zip(types).map {
case (n, t) => n + ": " + t}
.mkString(", ") + ")"
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
index f8a5d03..9d9a19f 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/unfinishedKeyPairOperation.scala
@@ -22,7 +22,7 @@ import org.apache.flink.api.common.InvalidProgramException
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.java.operators.Keys
-import org.apache.flink.api.java.operators.Keys.FieldPositionKeys
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys
import org.apache.flink.api.common.typeinfo.TypeInformation
/**
@@ -56,7 +56,7 @@ private[flink] abstract class UnfinishedKeyPairOperation[L, R, O](
* This only works on Tuple [[DataSet]].
*/
def where(leftKeys: Int*) = {
- val leftKey = new FieldPositionKeys[L](leftKeys.toArray, leftInput.getType)
+ val leftKey = new ExpressionKeys[L](leftKeys.toArray, leftInput.getType)
new HalfUnfinishedKeyPairOperation[L, R, O](this, leftKey)
}
@@ -73,7 +73,7 @@ private[flink] abstract class UnfinishedKeyPairOperation[L, R, O](
leftInput.getType,
firstLeftField +: otherLeftFields.toArray)
- val leftKey = new FieldPositionKeys[L](fieldIndices, leftInput.getType)
+ val leftKey = new ExpressionKeys[L](fieldIndices, leftInput.getType)
new HalfUnfinishedKeyPairOperation[L, R, O](this, leftKey)
}
@@ -103,8 +103,8 @@ private[flink] class HalfUnfinishedKeyPairOperation[L, R, O](
* This only works on a Tuple [[DataSet]].
*/
def equalTo(rightKeys: Int*): O = {
- val rightKey = new FieldPositionKeys[R](rightKeys.toArray, unfinished.rightInput.getType)
- if (!leftKey.areCompatibale(rightKey)) {
+ val rightKey = new ExpressionKeys[R](rightKeys.toArray, unfinished.rightInput.getType)
+ if (!leftKey.areCompatible(rightKey)) {
throw new InvalidProgramException("The types of the key fields do not match. Left: " +
leftKey + " Right: " + rightKey)
}
@@ -122,8 +122,8 @@ private[flink] class HalfUnfinishedKeyPairOperation[L, R, O](
unfinished.rightInput.getType,
firstRightField +: otherRightFields.toArray)
- val rightKey = new FieldPositionKeys[R](fieldIndices, unfinished.rightInput.getType)
- if (!leftKey.areCompatibale(rightKey)) {
+ val rightKey = new ExpressionKeys[R](fieldIndices, unfinished.rightInput.getType)
+ if (!leftKey.areCompatible(rightKey)) {
throw new InvalidProgramException("The types of the key fields do not match. Left: " +
leftKey + " Right: " + rightKey)
}
@@ -145,7 +145,7 @@ private[flink] class HalfUnfinishedKeyPairOperation[L, R, O](
unfinished.rightInput.getType,
keyType)
- if (!leftKey.areCompatibale(rightKey)) {
+ if (!leftKey.areCompatible(rightKey)) {
throw new InvalidProgramException("The types of the key fields do not match. Left: " +
leftKey + " Right: " + rightKey)
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
index a3bb34d..d962b76 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/CoGroupOperatorTest.scala
@@ -19,6 +19,7 @@ package org.apache.flink.api.scala.operators
import java.io.Serializable
import org.apache.flink.api.common.InvalidProgramException
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
import org.junit.Assert
import org.junit.Ignore
import org.junit.Test
@@ -44,7 +45,7 @@ class CoGroupOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyFields2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -54,7 +55,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where(0).equalTo(2)
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyFields3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -109,7 +110,7 @@ class CoGroupOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyFieldNames2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -119,7 +120,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where("_1").equalTo("_3")
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyFieldNames3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -254,7 +255,7 @@ class CoGroupOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyMixing3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -264,7 +265,7 @@ class CoGroupOperatorTest {
ds1.coGroup(ds2).where(2).equalTo { _.l }
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testCoGroupKeyMixing4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
index 05e45d8..cae936d 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/JoinOperatorTest.scala
@@ -17,6 +17,7 @@
*/
package org.apache.flink.api.scala.operators
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException
import org.junit.Assert
import org.apache.flink.api.common.InvalidProgramException
import org.junit.Ignore
@@ -45,7 +46,7 @@ class JoinOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyIndices2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -55,7 +56,7 @@ class JoinOperatorTest {
ds1.join(ds2).where(0).equalTo(2)
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyIndices3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -110,7 +111,7 @@ class JoinOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyFields2(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -120,7 +121,7 @@ class JoinOperatorTest {
ds1.join(ds2).where("_1").equalTo("_3")
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyFields3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -255,7 +256,7 @@ class JoinOperatorTest {
}
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyMixing3(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
@@ -265,7 +266,7 @@ class JoinOperatorTest {
ds1.join(ds2).where(2).equalTo { _.l }
}
- @Test(expected = classOf[InvalidProgramException])
+ @Test(expected = classOf[IncompatibleKeysException])
def testJoinKeyMixing4(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val ds1 = env.fromCollection(emptyTupleData)
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
index e810b45..adc46ac 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD2Test.scala
@@ -30,7 +30,7 @@ class TupleComparatorILD2Test extends TupleComparatorTestBase[(Int, Long, Double
protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
val ti = createTypeInformation[(Int, Long, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(0, 1), Array(ascending, ascending))
+ .createComparator(Array(0, 1), Array(ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
index 96cc42c..377f985 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILD3Test.scala
@@ -28,7 +28,7 @@ class TupleComparatorILD3Test extends TupleComparatorTestBase[(Int, Long, Double
protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
val ti = createTypeInformation[(Int, Long, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending))
+ .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
index 30d55c9..1578951 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDC3Test.scala
@@ -28,7 +28,7 @@ class TupleComparatorILDC3Test extends TupleComparatorTestBase[(Int, Long, Doubl
protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
val ti = createTypeInformation[(Int, Long, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(2, 0, 1), Array(ascending, ascending, ascending))
+ .createComparator(Array(2, 0, 1), Array(ascending, ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
index df0983c..51e08cc 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDX1Test.scala
@@ -28,7 +28,7 @@ class TupleComparatorILDX1Test extends TupleComparatorTestBase[(Int, Long, Doubl
protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
val ti = createTypeInformation[(Int, Long, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(1), Array(ascending))
+ .createComparator(Array(1), Array(ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
index c4e3883..3dbaabf 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorILDXC2Test.scala
@@ -28,7 +28,7 @@ class TupleComparatorILDXC2Test extends TupleComparatorTestBase[(Int, Long, Doub
protected def createComparator(ascending: Boolean): TypeComparator[(Int, Long, Double)] = {
val ti = createTypeInformation[(Int, Long, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, Long, Double)]]
- .createComparator(Array(2, 1), Array(ascending, ascending))
+ .createComparator(Array(2, 1), Array(ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, Long, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
index 01c3ab8..252ae79 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD1Test.scala
@@ -27,7 +27,7 @@ class TupleComparatorISD1Test extends TupleComparatorTestBase[(Int, String, Doub
protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
val ti = createTypeInformation[(Int, String, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0), Array(ascending))
+ .createComparator(Array(0), Array(ascending),0)
}
protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
index 20c1955..37e775e 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD2Test.scala
@@ -27,7 +27,7 @@ class TupleComparatorISD2Test extends TupleComparatorTestBase[(Int, String, Doub
protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
val ti = createTypeInformation[(Int, String, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0, 1), Array(ascending, ascending))
+ .createComparator(Array(0, 1), Array(ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
index 6d945a9..227b041 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/runtime/TupleComparatorISD3Test.scala
@@ -27,7 +27,7 @@ class TupleComparatorISD3Test extends TupleComparatorTestBase[(Int, String, Doub
protected def createComparator(ascending: Boolean): TypeComparator[(Int, String, Double)] = {
val ti = createTypeInformation[(Int, String, Double)]
ti.asInstanceOf[TupleTypeInfoBase[(Int, String, Double)]]
- .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending))
+ .createComparator(Array(0, 1, 2), Array(ascending, ascending, ascending), 0)
}
protected def createSerializer: TypeSerializer[(Int, String, Double)] = {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountNestedPOJOITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountNestedPOJOITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountNestedPOJOITCase.java
new file mode 100644
index 0000000..bdd5a7d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleJavaPrograms/WordCountNestedPOJOITCase.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.exampleJavaPrograms;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.util.JavaProgramTestBase;
+import org.apache.flink.util.Collector;
+
+import java.io.Serializable;
+import java.util.Date;
+
+
+public class WordCountNestedPOJOITCase extends JavaProgramTestBase implements Serializable {
+ private static final long serialVersionUID = 1L;
+ protected String textPath;
+ protected String resultPath;
+
+
+ @Override
+ protected void preSubmit() throws Exception {
+ textPath = createTempFile("text.txt", WordCountData.TEXT);
+ resultPath = getTempDirPath("result");
+ }
+
+ @Override
+ protected void postSubmit() throws Exception {
+ compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath);
+ }
+
+ @Override
+ protected void testProgram() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<String> text = env.readTextFile(textPath);
+
+ DataSet<WC> counts = text
+ .flatMap(new Tokenizer())
+ .groupBy("complex.someTest")
+ .reduce(new ReduceFunction<WC>() {
+ private static final long serialVersionUID = 1L;
+ public WC reduce(WC value1, WC value2) {
+ return new WC(value1.complex.someTest, value1.count + value2.count);
+ }
+ });
+
+ counts.writeAsText(resultPath);
+
+ env.execute("WordCount with custom data types example");
+ }
+
+ public static final class Tokenizer implements FlatMapFunction<String, WC> {
+
+ @Override
+ public void flatMap(String value, Collector<WC> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new WC(token, 1));
+ }
+ }
+ }
+ }
+
+ public static class WC { // is a pojo
+ public ComplexNestedClass complex; // is a pojo
+ public int count; // is a BasicType
+
+ public WC() {
+ }
+ public WC(String t, int c) {
+ this.count = c;
+ this.complex = new ComplexNestedClass();
+ this.complex.word = new Tuple3<Long, Long, String>(0L, 0L, "egal");
+ this.complex.date = new Date();
+ this.complex.someFloat = 0.0f;
+ this.complex.someNumber = 666;
+ this.complex.someTest = t;
+ }
+ @Override
+ public String toString() {
+ return this.complex.someTest+" "+count;
+ }
+ }
+
+ public static class ComplexNestedClass { // pojo
+ public static int ignoreStaticField;
+ public transient int ignoreTransientField;
+ public Date date; // generic type
+ public Integer someNumber; // BasicType
+ public float someFloat; // BasicType
+ public Tuple3<Long, Long, String> word; //Tuple Type with three basic types
+ public String someTest;
+ }
+
+}
[07/11] [FLINK-1032] Rework support for POJO types in the Java API
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
index 9ecb9a2..56f90f4 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/CoGroupOperator.java
@@ -32,7 +32,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.operators.DeltaIteration.SolutionSetPlaceHolder;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys.FieldPositionKeys;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanBothUnwrappingCoGroupOperator;
import org.apache.flink.api.java.operators.translation.PlanLeftUnwrappingCoGroupOperator;
@@ -42,6 +43,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+
/**
* A {@link DataSet} that is the result of a CoGroup transformation.
*
@@ -74,16 +76,16 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
// sanity check solution set key mismatches
if (input1 instanceof SolutionSetPlaceHolder) {
- if (keys1 instanceof FieldPositionKeys) {
- int[] positions = ((FieldPositionKeys<?>) keys1).computeLogicalKeyPositions();
+ if (keys1 instanceof ExpressionKeys) {
+ int[] positions = ((ExpressionKeys<?>) keys1).computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input1).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException("Currently, the solution set may only be CoGrouped with using tuple field positions.");
}
}
if (input2 instanceof SolutionSetPlaceHolder) {
- if (keys2 instanceof FieldPositionKeys) {
- int[] positions = ((FieldPositionKeys<?>) keys2).computeLogicalKeyPositions();
+ if (keys2 instanceof ExpressionKeys) {
+ int[] positions = ((ExpressionKeys<?>) keys2).computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input2).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException("Currently, the solution set may only be CoGrouped with using tuple field positions.");
@@ -108,10 +110,10 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
protected org.apache.flink.api.common.operators.base.CoGroupOperatorBase<?, ?, OUT, ?> translateToDataFlow(Operator<I1> input1, Operator<I2> input2) {
String name = getName() != null ? getName() : function.getClass().getName();
-
- if (!keys1.areCompatibale(keys2)) {
- throw new InvalidProgramException("The types of the key fields do not match. Left:" +
- " " + keys1 + " Right: " + keys2);
+ try {
+ keys1.areCompatible(keys2);
+ } catch (IncompatibleKeysException e) {
+ throw new InvalidProgramException("The types of the key fields do not match.", e);
}
if (keys1 instanceof Keys.SelectorFunctionKeys
@@ -166,15 +168,13 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
return po;
}
- else if ((keys1 instanceof Keys.FieldPositionKeys
- && keys2 instanceof Keys.FieldPositionKeys) ||
- ((keys1 instanceof Keys.ExpressionKeys
- && keys2 instanceof Keys.ExpressionKeys)))
+ else if ( keys1 instanceof Keys.ExpressionKeys && keys2 instanceof Keys.ExpressionKeys)
{
-
- if (!keys1.areCompatibale(keys2)) {
- throw new InvalidProgramException("The types of the key fields do not match.");
- }
+ try {
+ keys1.areCompatible(keys2);
+ } catch (IncompatibleKeysException e) {
+ throw new InvalidProgramException("The types of the key fields do not match.", e);
+ }
int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions();
int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
@@ -364,7 +364,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @see DataSet
*/
public CoGroupOperatorSetsPredicate where(int... fields) {
- return new CoGroupOperatorSetsPredicate(new Keys.FieldPositionKeys<I1>(fields, input1.getType()));
+ return new CoGroupOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
}
/**
@@ -380,9 +380,9 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @see Tuple
* @see DataSet
*/
-// public CoGroupOperatorSetsPredicate where(String... fields) {
-// return new CoGroupOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
-// }
+ public CoGroupOperatorSetsPredicate where(String... fields) {
+ return new CoGroupOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
+ }
/**
* Continues a CoGroup transformation and defines a {@link KeySelector} function for the first co-grouped {@link DataSet}.</br>
@@ -436,7 +436,7 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.common.functions.CoGroupFunction)} to finalize the CoGroup transformation.
*/
public CoGroupOperatorWithoutFunction equalTo(int... fields) {
- return createCoGroupOperator(new Keys.FieldPositionKeys<I2>(fields, input2.getType()));
+ return createCoGroupOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
}
/**
@@ -448,9 +448,9 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
* @return An incomplete CoGroup transformation.
* Call {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.common.functions.CoGroupFunction)} to finalize the CoGroup transformation.
*/
-// public CoGroupOperatorWithoutFunction equalTo(String... fields) {
-// return createCoGroupOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
-// }
+ public CoGroupOperatorWithoutFunction equalTo(String... fields) {
+ return createCoGroupOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
+ }
/**
* Continues a CoGroup transformation and defines a {@link KeySelector} function for the second co-grouped {@link DataSet}.</br>
@@ -480,9 +480,10 @@ public class CoGroupOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1, I2, OU
if (keys2.isEmpty()) {
throw new InvalidProgramException("The co-group keys must not be empty.");
}
-
- if (!keys1.areCompatibale(keys2)) {
- throw new InvalidProgramException("The pair of co-group keys are not compatible with each other.");
+ try {
+ keys1.areCompatible(keys2);
+ } catch(IncompatibleKeysException ike) {
+ throw new InvalidProgramException("The pair of co-group keys are not compatible with each other.", ike);
}
return new CoGroupOperatorWithoutFunction(keys2);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
index 18cb8f6..54a65a9 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/DistinctOperator.java
@@ -58,7 +58,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
for(int i = 0; i < tupleType.getArity(); i++) {
allFields[i] = i;
}
- keys = new Keys.FieldPositionKeys<T>(allFields, input.getType(), true);
+ keys = new Keys.ExpressionKeys<T>(allFields, input.getType(), true);
}
else {
throw new InvalidProgramException("Distinction on all fields is only possible on tuple data types.");
@@ -67,7 +67,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
// FieldPositionKeys can only be applied on Tuples
- if (keys instanceof Keys.FieldPositionKeys && !input.getType().isTupleType()) {
+ if (keys instanceof Keys.ExpressionKeys && !input.getType().isTupleType()) {
throw new InvalidProgramException("Distinction on field positions is only possible on tuple data types.");
}
@@ -81,7 +81,7 @@ public class DistinctOperator<T> extends SingleInputOperator<T, T, DistinctOpera
String name = function.getClass().getName();
- if (keys instanceof Keys.FieldPositionKeys) {
+ if (keys instanceof Keys.ExpressionKeys) {
int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<T, T>(getInputType(), getResultType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
index edae3c8..cbcc367 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
@@ -139,7 +139,8 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return po;
}
- else if (grouper.getKeys() instanceof Keys.FieldPositionKeys) {
+ else if (grouper.getKeys() instanceof Keys.ExpressionKeys) { //was field position key
+ //TODO ask stephan
int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
@@ -166,19 +167,19 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return po;
}
- else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
-
- int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
- UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
- GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
- new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
-
- po.setCombinable(combinable);
- po.setInput(input);
- po.setDegreeOfParallelism(this.getParallelism());
-
- return po;
- }
+// else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
+//
+// int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
+// UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
+// GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
+// new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
+//
+// po.setCombinable(combinable);
+// po.setInput(input);
+// po.setDegreeOfParallelism(this.getParallelism());
+//
+// return po;
+// }
else {
throw new UnsupportedOperationException("Unrecognized key type.");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
index bc35c14..caa27dc 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/JoinOperator.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichFlatJoinFunction;
import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.operators.DualInputSemanticProperties;
import org.apache.flink.api.common.operators.Operator;
@@ -33,23 +34,22 @@ import org.apache.flink.api.common.operators.base.JoinOperatorBase;
import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.DeltaIteration.SolutionSetPlaceHolder;
-import org.apache.flink.api.common.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.functions.SemanticPropUtil;
-import org.apache.flink.api.java.operators.Keys.FieldPositionKeys;
+import org.apache.flink.api.java.operators.DeltaIteration.SolutionSetPlaceHolder;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
+import org.apache.flink.api.java.operators.Keys.IncompatibleKeysException;
import org.apache.flink.api.java.operators.translation.KeyExtractingMapper;
import org.apache.flink.api.java.operators.translation.PlanBothUnwrappingJoinOperator;
import org.apache.flink.api.java.operators.translation.PlanLeftUnwrappingJoinOperator;
import org.apache.flink.api.java.operators.translation.PlanRightUnwrappingJoinOperator;
import org.apache.flink.api.java.operators.translation.WrappingFunction;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-
//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
import org.apache.flink.api.java.tuple.*;
-import org.apache.flink.util.Collector;
//CHECKSTYLE.ON: AvoidStarImport
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.util.Collector;
/**
* A {@link DataSet} that is the result of a Join transformation.
@@ -124,16 +124,16 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
// sanity check solution set key mismatches
if (input1 instanceof SolutionSetPlaceHolder) {
- if (keys1 instanceof FieldPositionKeys) {
- int[] positions = ((FieldPositionKeys<?>) keys1).computeLogicalKeyPositions();
+ if (keys1 instanceof ExpressionKeys) {
+ int[] positions = ((ExpressionKeys<?>) keys1).computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input1).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException("Currently, the solution set may only be joined with using tuple field positions.");
}
}
if (input2 instanceof SolutionSetPlaceHolder) {
- if (keys2 instanceof FieldPositionKeys) {
- int[] positions = ((FieldPositionKeys<?>) keys2).computeLogicalKeyPositions();
+ if (keys2 instanceof ExpressionKeys) {
+ int[] positions = ((ExpressionKeys<?>) keys2).computeLogicalKeyPositions();
((SolutionSetPlaceHolder<?>) input2).checkJoinKeyFields(positions);
} else {
throw new InvalidProgramException("Currently, the solution set may only be joined with using tuple field positions.");
@@ -247,12 +247,12 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
protected JoinOperatorBase<?, ?, OUT, ?> translateToDataFlow(
Operator<I1> input1,
Operator<I2> input2) {
-
- String name = getName() != null ? getName() : function.getClass().getName();
- if (!keys1.areCompatibale(keys2)) {
- throw new InvalidProgramException("The types of the key fields do not match. Left:" +
- " " + keys1 + " Right: " + keys2);
+ String name = getName() != null ? getName() : function.getClass().getName();
+ try {
+ keys1.areCompatible(super.keys2);
+ } catch(IncompatibleKeysException ike) {
+ throw new InvalidProgramException("The types of the key fields do not match.", ike);
}
if (keys1 instanceof Keys.SelectorFunctionKeys
@@ -315,10 +315,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
return po;
}
- else if ((super.keys1 instanceof Keys.FieldPositionKeys
- && super.keys2 instanceof Keys.FieldPositionKeys) ||
- ((super.keys1 instanceof Keys.ExpressionKeys
- && super.keys2 instanceof Keys.ExpressionKeys)))
+ else if (super.keys1 instanceof Keys.ExpressionKeys && super.keys2 instanceof Keys.ExpressionKeys)
{
// Neither side needs the tuple wrapping/unwrapping
@@ -765,7 +762,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @see DataSet
*/
public JoinOperatorSetsPredicate where(int... fields) {
- return new JoinOperatorSetsPredicate(new Keys.FieldPositionKeys<I1>(fields, input1.getType()));
+ return new JoinOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
}
/**
@@ -782,9 +779,9 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @see Tuple
* @see DataSet
*/
-// public JoinOperatorSetsPredicate where(String... fields) {
-// return new JoinOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
-// }
+ public JoinOperatorSetsPredicate where(String... fields) {
+ return new JoinOperatorSetsPredicate(new Keys.ExpressionKeys<I1>(fields, input1.getType()));
+ }
/**
* Continues a Join transformation and defines a {@link KeySelector} function for the first join {@link DataSet}.</br>
@@ -843,7 +840,7 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @return A DefaultJoin that represents the joined DataSet.
*/
public DefaultJoin<I1, I2> equalTo(int... fields) {
- return createJoinOperator(new Keys.FieldPositionKeys<I2>(fields, input2.getType()));
+ return createJoinOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
}
/**
@@ -857,9 +854,9 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
* @param fields The fields of the second join DataSet that should be used as keys.
* @return A DefaultJoin that represents the joined DataSet.
*/
-// public DefaultJoin<I1, I2> equalTo(String... fields) {
-// return createJoinOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
-// }
+ public DefaultJoin<I1, I2> equalTo(String... fields) {
+ return createJoinOperator(new Keys.ExpressionKeys<I2>(fields, input2.getType()));
+ }
/**
* Continues a Join transformation and defines a {@link KeySelector} function for the second join {@link DataSet}.</br>
@@ -887,8 +884,10 @@ public abstract class JoinOperator<I1, I2, OUT> extends TwoInputUdfOperator<I1,
throw new InvalidProgramException("The join keys may not be empty.");
}
- if (!keys1.areCompatibale(keys2)) {
- throw new InvalidProgramException("The pair of join keys are not compatible with each other.");
+ try {
+ keys1.areCompatible(keys2);
+ } catch (IncompatibleKeysException e) {
+ throw new InvalidProgramException("The pair of join keys are not compatible with each other.",e);
}
return new DefaultJoin<I1, I2>(input1, input2, keys1, keys2, joinHint);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
index b19d5c3..653a04a 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
@@ -18,116 +18,53 @@
package org.apache.flink.api.java.operators;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
-public abstract class Keys<T> {
+public abstract class Keys<T> {
+ private static final Logger LOG = LoggerFactory.getLogger(Keys.class);
public abstract int getNumberOfKeyFields();
public boolean isEmpty() {
return getNumberOfKeyFields() == 0;
}
-
- public abstract boolean areCompatibale(Keys<?> other);
-
+
+ /**
+ * Check if two sets of keys are compatible to each other (matching types, key counts)
+ */
+ public abstract boolean areCompatible(Keys<?> other) throws IncompatibleKeysException;
+
public abstract int[] computeLogicalKeyPositions();
-
+
+
// --------------------------------------------------------------------------------------------
- // Specializations for field indexed / expression-based / extractor-based grouping
+ // Specializations for expression-based / extractor-based grouping
// --------------------------------------------------------------------------------------------
-
- public static class FieldPositionKeys<T> extends Keys<T> {
-
- private final int[] fieldPositions;
- private final TypeInformation<?>[] types;
-
- public FieldPositionKeys(int[] groupingFields, TypeInformation<T> type) {
- this(groupingFields, type, false);
- }
-
- public FieldPositionKeys(int[] groupingFields, TypeInformation<T> type, boolean allowEmpty) {
- if (!type.isTupleType()) {
- throw new InvalidProgramException("Specifying keys via field positions is only valid" +
- "for tuple data types. Type: " + type);
- }
-
- if (!allowEmpty && (groupingFields == null || groupingFields.length == 0)) {
- throw new IllegalArgumentException("The grouping fields must not be empty.");
- }
-
- TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>)type;
-
- this.fieldPositions = makeFields(groupingFields, (TupleTypeInfoBase<?>) type);
-
- types = new TypeInformation[this.fieldPositions.length];
- for(int i = 0; i < this.fieldPositions.length; i++) {
- types[i] = tupleType.getTypeAt(this.fieldPositions[i]);
- }
-
- }
-
- @Override
- public int getNumberOfKeyFields() {
- return this.fieldPositions.length;
- }
-
- @Override
- public boolean areCompatibale(Keys<?> other) {
-
- if (other instanceof FieldPositionKeys) {
- FieldPositionKeys<?> oKey = (FieldPositionKeys<?>) other;
-
- if(oKey.types.length != this.types.length) {
- return false;
- }
- for(int i=0; i<this.types.length; i++) {
- if(!this.types[i].equals(oKey.types[i])) {
- return false;
- }
- }
- return true;
-
- } else if (other instanceof SelectorFunctionKeys) {
- if(this.types.length != 1) {
- return false;
- }
-
- SelectorFunctionKeys<?, ?> sfk = (SelectorFunctionKeys<?, ?>) other;
-
- return sfk.keyType.equals(this.types[0]);
- }
- else {
- return false;
- }
- }
-
- @Override
- public int[] computeLogicalKeyPositions() {
- return this.fieldPositions;
- }
-
- @Override
- public String toString() {
- String fieldsString = Arrays.toString(fieldPositions);
- String typesString = Arrays.toString(types);
- return "Tuple position key (Fields: " + fieldsString + " Types: " + typesString + ")";
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
+
+
public static class SelectorFunctionKeys<T, K> extends Keys<T> {
private final KeySelector<T, K> keyExtractor;
private final TypeInformation<K> keyType;
+ private final int[] logicalKeyFields;
public SelectorFunctionKeys(KeySelector<T, K> keyExtractor, TypeInformation<T> inputType, TypeInformation<K> keyType) {
if (keyExtractor == null) {
@@ -136,6 +73,15 @@ public abstract class Keys<T> {
this.keyExtractor = keyExtractor;
this.keyType = keyType;
+
+ // we have to handle a special case here:
+ // if the keyType is a tuple type, we need to select the full tuple with all its fields.
+ if(keyType.isTupleType()) {
+ ExpressionKeys<K> ek = new ExpressionKeys<K>(new String[] {ExpressionKeys.SELECT_ALL_CHAR}, keyType);
+ logicalKeyFields = ek.computeLogicalKeyPositions();
+ } else {
+ logicalKeyFields = new int[] {0};
+ }
if (!this.keyType.isKeyType()) {
throw new IllegalArgumentException("Invalid type of KeySelector keys");
@@ -152,35 +98,53 @@ public abstract class Keys<T> {
@Override
public int getNumberOfKeyFields() {
- return 1;
+ return logicalKeyFields.length;
}
@Override
- public boolean areCompatibale(Keys<?> other) {
-
+ public boolean areCompatible(Keys<?> other) throws IncompatibleKeysException {
+
if (other instanceof SelectorFunctionKeys) {
@SuppressWarnings("unchecked")
SelectorFunctionKeys<?, K> sfk = (SelectorFunctionKeys<?, K>) other;
return sfk.keyType.equals(this.keyType);
}
- else if (other instanceof FieldPositionKeys) {
- FieldPositionKeys<?> fpk = (FieldPositionKeys<?>) other;
-
- if(fpk.types.length != 1) {
- return false;
+ else if (other instanceof ExpressionKeys) {
+ ExpressionKeys<?> expressionKeys = (ExpressionKeys<?>) other;
+
+ if(keyType.isTupleType()) {
+ // special case again:
+ TupleTypeInfo<?> tupleKeyType = (TupleTypeInfo<?>) keyType;
+ List<FlatFieldDescriptor> keyTypeFields = new ArrayList<FlatFieldDescriptor>(tupleKeyType.getTotalFields());
+ tupleKeyType.getKey(ExpressionKeys.SELECT_ALL_CHAR, 0, keyTypeFields);
+ if(expressionKeys.keyFields.size() != keyTypeFields.size()) {
+ throw new IncompatibleKeysException(IncompatibleKeysException.SIZE_MISMATCH_MESSAGE);
+ }
+ for(int i=0; i < expressionKeys.keyFields.size(); i++) {
+ if(!expressionKeys.keyFields.get(i).getType().equals(keyTypeFields.get(i).getType())) {
+ throw new IncompatibleKeysException(expressionKeys.keyFields.get(i).getType(), keyTypeFields.get(i).getType() );
+ }
+ }
+ return true;
}
-
- return fpk.types[0].equals(this.keyType);
- }
- else {
- return false;
+ if(expressionKeys.getNumberOfKeyFields() != 1) {
+ throw new IncompatibleKeysException("Key selector functions are only compatible to one key");
+ }
+
+ if(expressionKeys.keyFields.get(0).getType().equals(this.keyType)) {
+ return true;
+ } else {
+ throw new IncompatibleKeysException(expressionKeys.keyFields.get(0).getType(), this.keyType);
+ }
+ } else {
+ throw new IncompatibleKeysException("The key is not compatible with "+other);
}
}
@Override
public int[] computeLogicalKeyPositions() {
- return new int[] {0};
+ return logicalKeyFields;
}
@Override
@@ -188,93 +152,178 @@ public abstract class Keys<T> {
return "Key function (Type: " + keyType + ")";
}
}
-
- // --------------------------------------------------------------------------------------------
-
+
+
+ /**
+ * Represents (nested) field access through string and integer-based keys for Composite Types (Tuple or Pojo)
+ */
public static class ExpressionKeys<T> extends Keys<T> {
+
+ public static final String SELECT_ALL_CHAR = "*";
+
+ /**
+ * Flattened fields representing keys fields
+ */
+ private List<FlatFieldDescriptor> keyFields;
+
+ /**
+ * two constructors for field-based (tuple-type) keys
+ */
+ public ExpressionKeys(int[] groupingFields, TypeInformation<T> type) {
+ this(groupingFields, type, false);
+ }
- private int[] logicalPositions;
-
- private final TypeInformation<?>[] types;
-
- @SuppressWarnings("unused")
- private PojoTypeInfo<?> type;
+ // int-defined field
+ public ExpressionKeys(int[] groupingFields, TypeInformation<T> type, boolean allowEmpty) {
+ if (!type.isTupleType()) {
+ throw new InvalidProgramException("Specifying keys via field positions is only valid" +
+ "for tuple data types. Type: " + type);
+ }
- public ExpressionKeys(String[] expressions, TypeInformation<T> type) {
- if (!(type instanceof PojoTypeInfo<?>)) {
- throw new UnsupportedOperationException("Key expressions can only be used on POJOs." + " " +
- "A POJO must have a default constructor without arguments and not have readObject" +
- " and/or writeObject methods. A current restriction is that it can only have nested POJOs or primitive (also boxed)" +
- " fields.");
+ if (!allowEmpty && (groupingFields == null || groupingFields.length == 0)) {
+ throw new IllegalArgumentException("The grouping fields must not be empty.");
}
- PojoTypeInfo<?> pojoType = (PojoTypeInfo<?>) type;
- this.type = pojoType;
- logicalPositions = pojoType.getLogicalPositions(expressions);
- types = pojoType.getTypes(expressions);
-
- for (int i = 0; i < logicalPositions.length; i++) {
- if (logicalPositions[i] < 0) {
- throw new IllegalArgumentException("Expression '" + expressions[i] + "' is not a valid key for POJO" +
- " type " + type.toString() + ".");
+ // select all fields. Therefore, set all fields on this tuple level and let the logic handle the rest
+ // (makes type assignment easier).
+ if (groupingFields == null || groupingFields.length == 0) {
+ groupingFields = new int[type.getArity()];
+ for (int i = 0; i < groupingFields.length; i++) {
+ groupingFields[i] = i;
}
+ } else {
+ groupingFields = rangeCheckFields(groupingFields, type.getArity() -1);
}
+ TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>)type;
+ Preconditions.checkArgument(groupingFields.length > 0, "Grouping fields can not be empty at this point");
+
+ keyFields = new ArrayList<FlatFieldDescriptor>(type.getTotalFields());
+ // for each key, find the field:
+ for(int j = 0; j < groupingFields.length; j++) {
+ for(int i = 0; i < type.getArity(); i++) {
+ TypeInformation<?> fieldType = tupleType.getTypeAt(i);
+
+ if(groupingFields[j] == i) { // check if user set the key
+ int keyId = countNestedElementsBefore(tupleType, i) + i;
+ if(fieldType instanceof TupleTypeInfoBase) {
+ TupleTypeInfoBase<?> tupleFieldType = (TupleTypeInfoBase<?>) fieldType;
+ tupleFieldType.addAllFields(keyId, keyFields);
+ } else {
+ Preconditions.checkArgument(fieldType instanceof AtomicType, "Wrong field type");
+ keyFields.add(new FlatFieldDescriptor(keyId, fieldType));
+ }
+
+ }
+ }
+ }
+ keyFields = removeNullElementsFromList(keyFields);
}
-
+
+ private static int countNestedElementsBefore(TupleTypeInfoBase<?> tupleType, int pos) {
+ if( pos == 0) {
+ return 0;
+ }
+ int ret = 0;
+ for (int i = 0; i < pos; i++) {
+ TypeInformation<?> fieldType = tupleType.getTypeAt(i);
+ ret += fieldType.getTotalFields() -1;
+ }
+ return ret;
+ }
+
+ public static <R> List<R> removeNullElementsFromList(List<R> in) {
+ List<R> elements = new ArrayList<R>();
+ for(R e: in) {
+ if(e != null) {
+ elements.add(e);
+ }
+ }
+ return elements;
+ }
+
+ /**
+ * Create ExpressionKeys from String-expressions
+ */
+ public ExpressionKeys(String[] expressionsIn, TypeInformation<T> type) {
+ if(!(type instanceof CompositeType<?>)) {
+ throw new IllegalArgumentException("Type "+type+" is not a composite type. Key expressions are not supported.");
+ }
+ CompositeType<T> cType = (CompositeType<T>) type;
+
+ String[] expressions = removeDuplicates(expressionsIn);
+ if(expressionsIn.length != expressions.length) {
+ LOG.warn("The key expressions contained duplicates. They are now unique");
+ }
+ // extract the keys on their flat position
+ keyFields = new ArrayList<FlatFieldDescriptor>(expressions.length);
+ for (int i = 0; i < expressions.length; i++) {
+ List<FlatFieldDescriptor> keys = new ArrayList<FlatFieldDescriptor>(); // use separate list to do a size check
+ cType.getKey(expressions[i], 0, keys);
+ if(keys.size() == 0) {
+ throw new IllegalArgumentException("Unable to extract key from expression "+expressions[i]+" on key "+cType);
+ }
+ keyFields.addAll(keys);
+ }
+ }
+
@Override
public int getNumberOfKeyFields() {
- return logicalPositions.length;
+ if(keyFields == null) {
+ return 0;
+ }
+ return keyFields.size();
}
@Override
- public boolean areCompatibale(Keys<?> other) {
+ public boolean areCompatible(Keys<?> other) throws IncompatibleKeysException {
if (other instanceof ExpressionKeys) {
ExpressionKeys<?> oKey = (ExpressionKeys<?>) other;
- if(oKey.types.length != this.types.length) {
- return false;
+ if(oKey.getNumberOfKeyFields() != this.getNumberOfKeyFields() ) {
+ throw new IncompatibleKeysException(IncompatibleKeysException.SIZE_MISMATCH_MESSAGE);
}
- for(int i=0; i<this.types.length; i++) {
- if(!this.types[i].equals(oKey.types[i])) {
- return false;
+ for(int i=0; i < this.keyFields.size(); i++) {
+ if(!this.keyFields.get(i).getType().equals(oKey.keyFields.get(i).getType())) {
+ throw new IncompatibleKeysException(this.keyFields.get(i).getType(), oKey.keyFields.get(i).getType() );
}
}
return true;
-
+ } else if(other instanceof SelectorFunctionKeys<?, ?>) {
+ return other.areCompatible(this);
} else {
- return false;
+ throw new IncompatibleKeysException("The key is not compatible with "+other);
}
}
@Override
public int[] computeLogicalKeyPositions() {
- return logicalPositions;
+ List<Integer> logicalKeys = new LinkedList<Integer>();
+ for(FlatFieldDescriptor kd : keyFields) {
+ logicalKeys.addAll( Ints.asList(kd.getPosition()));
+ }
+ return Ints.toArray(logicalKeys);
}
+
+ }
+
+ private static String[] removeDuplicates(String[] in) {
+ List<String> ret = new LinkedList<String>();
+ for(String el : in) {
+ if(!ret.contains(el)) {
+ ret.add(el);
+ }
+ }
+ return ret.toArray(new String[ret.size()]);
}
-
-
+ // --------------------------------------------------------------------------------------------
+
+
// --------------------------------------------------------------------------------------------
// Utilities
// --------------------------------------------------------------------------------------------
- private static int[] makeFields(int[] fields, TupleTypeInfoBase<?> type) {
- int inLength = type.getArity();
-
- // null parameter means all fields are considered
- if (fields == null || fields.length == 0) {
- fields = new int[inLength];
- for (int i = 0; i < inLength; i++) {
- fields[i] = i;
- }
- return fields;
- } else {
- return rangeCheckAndOrderFields(fields, inLength-1);
- }
- }
- private static final int[] rangeCheckAndOrderFields(int[] fields, int maxAllowedField) {
- // order
- Arrays.sort(fields);
+ private static final int[] rangeCheckFields(int[] fields, int maxAllowedField) {
// range check and duplicate eliminate
int i = 1, k = 0;
@@ -285,12 +334,12 @@ public abstract class Keys<T> {
}
for (; i < fields.length; i++) {
- if (fields[i] < 0 || i > maxAllowedField) {
+ if (fields[i] < 0 || fields[i] > maxAllowedField) {
throw new IllegalArgumentException("Tuple position is out of range.");
}
-
if (fields[i] != last) {
k++;
+ last = fields[i];
fields[k] = fields[i];
}
}
@@ -299,7 +348,20 @@ public abstract class Keys<T> {
if (k == fields.length - 1) {
return fields;
} else {
- return Arrays.copyOfRange(fields, 0, k);
+ return Arrays.copyOfRange(fields, 0, k+1);
+ }
+ }
+
+ public static class IncompatibleKeysException extends Exception {
+ private static final long serialVersionUID = 1L;
+ public static final String SIZE_MISMATCH_MESSAGE = "The number of specified keys is different.";
+
+ public IncompatibleKeysException(String message) {
+ super(message);
+ }
+
+ public IncompatibleKeysException(TypeInformation<?> typeInformation, TypeInformation<?> typeInformation2) {
+ super(typeInformation+" and "+typeInformation2+" are not compatible");
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index f0931b5..532e464 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -50,7 +50,7 @@ public class PartitionOperator<T> extends SingleInputUdfOperator<T, T, Partition
throw new UnsupportedOperationException("Range Partitioning not yet supported");
}
- if(pKeys instanceof Keys.FieldPositionKeys<?> && !input.getType().isTupleType()) {
+ if(pKeys instanceof Keys.ExpressionKeys<?> && !input.getType().isTupleType()) {
throw new IllegalArgumentException("Hash Partitioning with key fields only possible on Tuple DataSets");
}
@@ -83,7 +83,7 @@ public class PartitionOperator<T> extends SingleInputUdfOperator<T, T, Partition
}
else if (pMethod == PartitionMethod.HASH) {
- if (pKeys instanceof Keys.FieldPositionKeys) {
+ if (pKeys instanceof Keys.ExpressionKeys) {
int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();
UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<T, T>(getType(), getType());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
index 463b31c..8cb64ba 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/ReduceOperator.java
@@ -101,8 +101,7 @@ public class ReduceOperator<IN> extends SingleInputUdfOperator<IN, IN, ReduceOpe
MapOperatorBase<?, IN, ?> po = translateSelectorFunctionReducer(selectorKeys, function, getInputType(), name, input, this.getParallelism());
return po;
}
- else if (grouper.getKeys() instanceof Keys.FieldPositionKeys ||
- grouper.getKeys() instanceof Keys.ExpressionKeys) {
+ else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
// reduce with field positions
int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
index e862b5a..6272538 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
@@ -52,6 +52,11 @@ public class GenericTypeInfo<T> extends TypeInformation<T> implements AtomicType
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@Override
public Class<T> getTypeClass() {
@@ -65,6 +70,9 @@ public class GenericTypeInfo<T> extends TypeInformation<T> implements AtomicType
@Override
public TypeSerializer<T> createSerializer() {
+ // NOTE: The TypeExtractor / pojo logic is assuming that we are using a Avro Serializer here
+ // in particular classes implementing GenericContainer are handled as GenericTypeInfos
+ // (this will probably not work with Kryo)
return new AvroSerializer<T>(this.typeClass);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
index 226557f..55128e6 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ObjectArrayTypeInfo.java
@@ -64,6 +64,11 @@ public class ObjectArrayTypeInfo<T, C> extends TypeInformation<T> {
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@SuppressWarnings("unchecked")
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
index 1b8ef35..bf0e25a 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoField.java
@@ -22,7 +22,7 @@ import java.lang.reflect.Field;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-class PojoField {
+public class PojoField {
public Field field;
public TypeInformation<?> type;
@@ -30,4 +30,9 @@ class PojoField {
this.field = field;
this.type = type;
}
-}
+
+ @Override
+ public String toString() {
+ return "PojoField " + field.getDeclaringClass() + "." + field.getName() + " (" + type + ")";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
index 51ed507..fba1f24 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
@@ -18,33 +18,41 @@
package org.apache.flink.api.java.typeutils;
-import com.google.common.base.Joiner;
-
import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.typeinfo.AtomicType;
-import org.apache.flink.api.common.typeinfo.CompositeType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
+import com.google.common.base.Joiner;
+
/**
- *
+ * TypeInformation for arbitrary (they have to be java-beans-style) java objects (what we call POJO).
+ *
*/
-public class PojoTypeInfo<T> extends TypeInformation<T> implements CompositeType<T> {
+public class PojoTypeInfo<T> extends CompositeType<T>{
private final Class<T> typeClass;
private PojoField[] fields;
+
+ private int totalFields;
public PojoTypeInfo(Class<T> typeClass, List<PojoField> fields) {
+ super(typeClass);
this.typeClass = typeClass;
List<PojoField> tempFields = new ArrayList<PojoField>(fields);
Collections.sort(tempFields, new Comparator<PojoField>() {
@@ -54,6 +62,14 @@ public class PojoTypeInfo<T> extends TypeInformation<T> implements CompositeType
}
});
this.fields = tempFields.toArray(new PojoField[tempFields.size()]);
+
+ // check if POJO is public
+ if(!Modifier.isPublic(typeClass.getModifiers())) {
+ throw new RuntimeException("POJO "+typeClass+" is not public");
+ }
+ for(PojoField field : fields) {
+ totalFields += field.type.getTotalFields();
+ }
}
@Override
@@ -71,6 +87,11 @@ public class PojoTypeInfo<T> extends TypeInformation<T> implements CompositeType
public int getArity() {
return fields.length;
}
+
+ @Override
+ public int getTotalFields() {
+ return totalFields;
+ }
@Override
public Class<T> getTypeClass() {
@@ -82,18 +103,6 @@ public class PojoTypeInfo<T> extends TypeInformation<T> implements CompositeType
return Comparable.class.isAssignableFrom(typeClass);
}
- @Override
- public TypeSerializer<T> createSerializer() {
- TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[fields.length];
- Field[] reflectiveFields = new Field[fields.length];
-
- for (int i = 0; i < fields.length; i++) {
- fieldSerializers[i] = fields[i].type.createSerializer();
- reflectiveFields[i] = fields[i].field;
- }
-
- return new PojoSerializer<T>(this.typeClass, fieldSerializers, reflectiveFields);
- }
@Override
public String toString() {
@@ -105,73 +114,124 @@ public class PojoTypeInfo<T> extends TypeInformation<T> implements CompositeType
+ ", fields = [" + Joiner.on(", ").join(fieldStrings) + "]"
+ ">";
}
-
- public int getLogicalPosition(String fieldExpression) {
- for (int i = 0; i < fields.length; i++) {
- if (fields[i].field.getName().equals(fieldExpression)) {
- return i;
+
+ @Override
+ public void getKey(String fieldExpression, int offset, List<FlatFieldDescriptor> result) {
+ // handle 'select all' first
+ if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR)) {
+ int keyPosition = 0;
+ for(PojoField field : fields) {
+ if(field.type instanceof AtomicType) {
+ result.add(new FlatFieldDescriptor(offset + keyPosition, field.type));
+ } else if(field.type instanceof CompositeType) {
+ CompositeType<?> cType = (CompositeType<?>)field.type;
+ cType.getKey(String.valueOf(ExpressionKeys.SELECT_ALL_CHAR), offset + keyPosition, result);
+ keyPosition += cType.getTotalFields()-1;
+ } else {
+ throw new RuntimeException("Unexpected key type: "+field.type);
+ }
+ keyPosition++;
}
+ return;
+ }
+ Validate.notEmpty(fieldExpression, "Field expression must not be empty.");
+ // if there is a dot try getting the field from that sub field
+ int firstDot = fieldExpression.indexOf('.');
+ if (firstDot == -1) {
+ // this is the last field (or only field) in the field expression
+ int fieldId = 0;
+ for (int i = 0; i < fields.length; i++) {
+ if(fields[i].type instanceof CompositeType) {
+ fieldId += fields[i].type.getTotalFields()-1;
+ }
+ if (fields[i].field.getName().equals(fieldExpression)) {
+ result.add(new FlatFieldDescriptor(offset + fieldId, fields[i].type));
+ return;
+ }
+ fieldId++;
+ }
+ } else {
+ // split and go deeper
+ String firstField = fieldExpression.substring(0, firstDot);
+ String rest = fieldExpression.substring(firstDot + 1);
+ int fieldId = 0;
+ for (int i = 0; i < fields.length; i++) {
+ if (fields[i].field.getName().equals(firstField)) {
+ if (!(fields[i].type instanceof CompositeType<?>)) {
+ throw new RuntimeException("Field "+fields[i].type+" is not composite type");
+ }
+ CompositeType<?> cType = (CompositeType<?>) fields[i].type;
+ cType.getKey(rest, offset + fieldId, result); // recurse
+ return;
+ }
+ fieldId++;
+ }
+ throw new RuntimeException("Unable to find field "+fieldExpression+" in type "+this+" (looking for '"+firstField+"')");
}
- return -1;
}
- public int[] getLogicalPositions(String[] fieldExpression) {
- int[] result = new int[fieldExpression.length];
- for (int i = 0; i < fieldExpression.length; i++) {
- result[i] = getLogicalPosition(fieldExpression[i]);
+ @Override
+ public <X> TypeInformation<X> getTypeAt(int pos) {
+ if (pos < 0 || pos >= this.fields.length) {
+ throw new IndexOutOfBoundsException();
}
- return result;
+ @SuppressWarnings("unchecked")
+ TypeInformation<X> typed = (TypeInformation<X>) fields[pos].type;
+ return typed;
}
- public TypeInformation<?> getType(String fieldExpression) {
- for (int i = 0; i < fields.length; i++) {
- if (fields[i].field.getName().equals(fieldExpression)) {
- return fields[i].type;
- }
+ // used for testing. Maybe use mockito here
+ public PojoField getPojoFieldAt(int pos) {
+ if (pos < 0 || pos >= this.fields.length) {
+ throw new IndexOutOfBoundsException();
}
- return null;
+ return this.fields[pos];
}
- public TypeInformation<?>[] getTypes(String[] fieldExpression) {
- TypeInformation<?>[] result = new TypeInformation<?>[fieldExpression.length];
- for (int i = 0; i < fieldExpression.length; i++) {
- result[i] = getType(fieldExpression[i]);
- }
- return result;
+ /**
+ * Comparator creation
+ */
+ private TypeComparator<?>[] fieldComparators;
+ private Field[] keyFields;
+ private int comparatorHelperIndex = 0;
+ @Override
+ protected void initializeNewComparator(int keyCount) {
+ fieldComparators = new TypeComparator<?>[keyCount];
+ keyFields = new Field[keyCount];
+ comparatorHelperIndex = 0;
}
@Override
- public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders) {
- // sanity checks
- if (logicalKeyFields == null || orders == null || logicalKeyFields.length != orders.length ||
- logicalKeyFields.length > fields.length)
- {
- throw new IllegalArgumentException();
- }
+ protected void addCompareField(int fieldId, TypeComparator<?> comparator) {
+ fieldComparators[comparatorHelperIndex] = comparator;
+ keyFields[comparatorHelperIndex] = fields[fieldId].field;
+ comparatorHelperIndex++;
+ }
-// if (logicalKeyFields.length == 1) {
-// return createSinglefieldComparator(logicalKeyFields[0], orders[0], types[logicalKeyFields[0]]);
-// }
+ @Override
+ protected TypeComparator<T> getNewComparator() {
+ // first remove the null array fields
+ final Field[] finalKeyFields = Arrays.copyOf(keyFields, comparatorHelperIndex);
+ @SuppressWarnings("rawtypes")
+ final TypeComparator[] finalFieldComparators = Arrays.copyOf(fieldComparators, comparatorHelperIndex);
+ if(finalFieldComparators.length == 0 || finalKeyFields.length == 0 || finalFieldComparators.length != finalKeyFields.length) {
+ throw new IllegalArgumentException("Pojo comparator creation has a bug");
+ }
+ return new PojoComparator<T>(finalKeyFields, finalFieldComparators, createSerializer(), typeClass);
+ }
- // create the comparators for the individual fields
- TypeComparator<?>[] fieldComparators = new TypeComparator<?>[logicalKeyFields.length];
- Field[] keyFields = new Field[logicalKeyFields.length];
- for (int i = 0; i < logicalKeyFields.length; i++) {
- int field = logicalKeyFields[i];
+ @Override
+ public TypeSerializer<T> createSerializer() {
+ TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[fields.length ];
+ Field[] reflectiveFields = new Field[fields.length];
- if (field < 0 || field >= fields.length) {
- throw new IllegalArgumentException("The field position " + field + " is out of range [0," + fields.length + ")");
- }
- if (fields[field].type.isKeyType() && fields[field].type instanceof AtomicType) {
- fieldComparators[i] = ((AtomicType<?>) fields[field].type).createComparator(orders[i]);
- keyFields[i] = fields[field].field;
- keyFields[i].setAccessible(true);
- } else {
- throw new IllegalArgumentException("The field at position " + field + " (" + fields[field].type + ") is no atomic key type.");
- }
+ for (int i = 0; i < fields.length; i++) {
+ fieldSerializers[i] = fields[i].type.createSerializer();
+ reflectiveFields[i] = fields[i].field;
}
- return new PojoComparator<T>(keyFields, fieldComparators, createSerializer(), typeClass);
+ return new PojoSerializer<T>(this.typeClass, fieldSerializers, reflectiveFields);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/RecordTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/RecordTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/RecordTypeInfo.java
index f069eed..2464f25 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/RecordTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/RecordTypeInfo.java
@@ -43,6 +43,11 @@ public class RecordTypeInfo extends TypeInformation<Record> {
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@Override
public Class<Record> getTypeClass() {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
index 6edb08c..82f9c50 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfo.java
@@ -20,22 +20,21 @@ package org.apache.flink.api.java.typeutils;
import java.util.Arrays;
-import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.runtime.TupleComparator;
-import org.apache.flink.api.java.typeutils.runtime.TupleSerializer;
-
//CHECKSTYLE.OFF: AvoidStarImport - Needed for TupleGenerator
import org.apache.flink.api.java.tuple.*;
//CHECKSTYLE.ON: AvoidStarImport
+import org.apache.flink.api.java.typeutils.runtime.TupleComparator;
+import org.apache.flink.api.java.typeutils.runtime.TupleSerializer;
-public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
+public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
+
@SuppressWarnings("unchecked")
public TupleTypeInfo(TypeInformation<?>... types) {
this((Class<T>) CLASSES[types.length - 1], types);
@@ -60,59 +59,72 @@ public final class TupleTypeInfo<T extends Tuple> extends TupleTypeInfoBase<T> {
return new TupleSerializer<T>(tupleClass, fieldSerializers);
}
+ /**
+ * Comparator creation
+ */
+ private TypeSerializer<?>[] fieldSerializers;
+ private TypeComparator<?>[] fieldComparators;
+ private int[] logicalKeyFields;
+ private int comparatorHelperIndex = 0;
+
@Override
- public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders) {
- // sanity checks
- if (logicalKeyFields == null || orders == null || logicalKeyFields.length != orders.length ||
- logicalKeyFields.length > types.length)
- {
- throw new IllegalArgumentException();
- }
+ protected void initializeNewComparator(int localKeyCount) {
+ fieldSerializers = new TypeSerializer[localKeyCount];
+ fieldComparators = new TypeComparator<?>[localKeyCount];
+ logicalKeyFields = new int[localKeyCount];
+ comparatorHelperIndex = 0;
+ }
- int maxKey = -1;
- for (int key : logicalKeyFields){
- maxKey = Math.max(key, maxKey);
- }
-
- if (maxKey >= this.types.length) {
- throw new IllegalArgumentException("The key position " + maxKey + " is out of range for Tuple" + types.length);
- }
-
- // create the comparators for the individual fields
- TypeComparator<?>[] fieldComparators = new TypeComparator<?>[logicalKeyFields.length];
- for (int i = 0; i < logicalKeyFields.length; i++) {
- int keyPos = logicalKeyFields[i];
- if (types[keyPos].isKeyType() && types[keyPos] instanceof AtomicType) {
- fieldComparators[i] = ((AtomicType<?>) types[keyPos]).createComparator(orders[i]);
- } else if(types[keyPos].isTupleType() && types[keyPos] instanceof TupleTypeInfo){ // Check for tuple
- TupleTypeInfo<?> tupleType = (TupleTypeInfo<?>) types[keyPos];
-
- // All fields are key
- int[] allFieldsKey = new int[tupleType.types.length];
- for(int h = 0; h < tupleType.types.length; h++){
- allFieldsKey[h]=h;
- }
-
- // Prepare order
- boolean[] tupleOrders = new boolean[tupleType.types.length];
- Arrays.fill(tupleOrders, orders[i]);
- fieldComparators[i] = tupleType.createComparator(allFieldsKey, tupleOrders);
- } else {
- throw new IllegalArgumentException("The field at position " + i + " (" + types[keyPos] + ") is no atomic key type nor tuple type.");
- }
- }
-
+ @Override
+ protected void addCompareField(int fieldId, TypeComparator<?> comparator) {
+ fieldComparators[comparatorHelperIndex] = comparator;
+ fieldSerializers[comparatorHelperIndex] = types[fieldId].createSerializer();
+ logicalKeyFields[comparatorHelperIndex] = fieldId;
+ comparatorHelperIndex++;
+ }
+
+ @Override
+ protected TypeComparator<T> getNewComparator() {
+ @SuppressWarnings("rawtypes")
+ final TypeComparator[] finalFieldComparators = Arrays.copyOf(fieldComparators, comparatorHelperIndex);
+ final int[] finalLogicalKeyFields = Arrays.copyOf(logicalKeyFields, comparatorHelperIndex);
+ //final TypeSerializer[] finalFieldSerializers = Arrays.copyOf(fieldSerializers, comparatorHelperIndex);
// create the serializers for the prefix up to highest key position
+ int maxKey = 0;
+ for(int key : finalLogicalKeyFields) {
+ maxKey = Math.max(maxKey, key);
+ }
TypeSerializer<?>[] fieldSerializers = new TypeSerializer<?>[maxKey + 1];
for (int i = 0; i <= maxKey; i++) {
fieldSerializers[i] = types[i].createSerializer();
}
-
- return new TupleComparator<T>(logicalKeyFields, fieldComparators, fieldSerializers);
+ if(finalFieldComparators.length == 0 || finalLogicalKeyFields.length == 0 || fieldSerializers.length == 0
+ || finalFieldComparators.length != finalLogicalKeyFields.length) {
+ throw new IllegalArgumentException("Tuple comparator creation has a bug");
+ }
+ return new TupleComparator<T>(finalLogicalKeyFields, finalFieldComparators, fieldSerializers);
}
// --------------------------------------------------------------------------------------------
-
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof TupleTypeInfo) {
+ @SuppressWarnings("unchecked")
+ TupleTypeInfo<T> other = (TupleTypeInfo<T>) obj;
+ return ((this.tupleType == null && other.tupleType == null) || this.tupleType.equals(other.tupleType)) &&
+ Arrays.deepEquals(this.types, other.types);
+
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return this.types.hashCode() ^ Arrays.deepHashCode(this.types);
+ }
+
@Override
public String toString() {
return "Java " + super.toString();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
index 3e1b646..4babbd7 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
@@ -19,19 +19,31 @@
package org.apache.flink.api.java.typeutils;
import java.util.Arrays;
+import java.util.List;
-import org.apache.flink.api.common.typeinfo.CompositeType;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
-public abstract class TupleTypeInfoBase<T> extends TypeInformation<T> implements CompositeType<T> {
+import com.google.common.base.Preconditions;
+
+public abstract class TupleTypeInfoBase<T> extends CompositeType<T> {
protected final TypeInformation<?>[] types;
protected final Class<T> tupleType;
+
+ private int totalFields;
public TupleTypeInfoBase(Class<T> tupleType, TypeInformation<?>... types) {
+ super(tupleType);
this.tupleType = tupleType;
this.types = types;
+ for(TypeInformation<?> type : types) {
+ totalFields += type.getTotalFields();
+ }
}
@Override
@@ -48,6 +60,11 @@ public abstract class TupleTypeInfoBase<T> extends TypeInformation<T> implements
public int getArity() {
return types.length;
}
+
+ @Override
+ public int getTotalFields() {
+ return totalFields;
+ }
@Override
public Class<T> getTypeClass() {
@@ -55,6 +72,95 @@ public abstract class TupleTypeInfoBase<T> extends TypeInformation<T> implements
}
+ /**
+ * Recursively add all fields in this tuple type. We need this in particular to get all
+ * the types.
+ * @param keyId
+ * @param keyFields
+ */
+ public void addAllFields(int startKeyId, List<FlatFieldDescriptor> keyFields) {
+ for(int i = 0; i < this.getArity(); i++) {
+ TypeInformation<?> type = this.types[i];
+ if(type instanceof AtomicType) {
+ keyFields.add(new FlatFieldDescriptor(startKeyId, type));
+ } else if(type instanceof TupleTypeInfoBase<?>) {
+ TupleTypeInfoBase<?> ttb = (TupleTypeInfoBase<?>) type;
+ ttb.addAllFields(startKeyId, keyFields);
+ }
+ startKeyId += type.getTotalFields();
+ }
+ }
+
+
+ @Override
+ public void getKey(String fieldExpression, int offset, List<FlatFieldDescriptor> result) {
+ // handle 'select all'
+ if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR)) {
+ int keyPosition = 0;
+ for(TypeInformation<?> type : types) {
+ if(type instanceof AtomicType) {
+ result.add(new FlatFieldDescriptor(offset + keyPosition, type));
+ } else if(type instanceof CompositeType) {
+ CompositeType<?> cType = (CompositeType<?>)type;
+ cType.getKey(String.valueOf(ExpressionKeys.SELECT_ALL_CHAR), offset + keyPosition, result);
+ keyPosition += cType.getTotalFields()-1;
+ } else {
+ throw new RuntimeException("Unexpected key type: "+type);
+ }
+ keyPosition++;
+ }
+ return;
+ }
+ // check input
+ if(fieldExpression.length() < 2) {
+ throw new IllegalArgumentException("The field expression '"+fieldExpression+"' is incorrect. The length must be at least 2");
+ }
+ if(fieldExpression.charAt(0) != 'f') {
+ throw new IllegalArgumentException("The field expression '"+fieldExpression+"' is incorrect for a Tuple type. It has to start with an 'f'");
+ }
+ // get first component of nested expression
+ int dotPos = fieldExpression.indexOf('.');
+ String nestedSplitFirst = fieldExpression;
+ if(dotPos != -1 ) {
+ Preconditions.checkArgument(dotPos != fieldExpression.length()-1, "The field expression can never end with a dot.");
+ nestedSplitFirst = fieldExpression.substring(0, dotPos);
+ }
+ String fieldNumStr = nestedSplitFirst.substring(1, nestedSplitFirst.length());
+ if(!StringUtils.isNumeric(fieldNumStr)) {
+ throw new IllegalArgumentException("The field expression '"+fieldExpression+"' is incorrect. Field number '"+fieldNumStr+" is not numeric");
+ }
+ int pos = -1;
+ try {
+ pos = Integer.valueOf(fieldNumStr);
+ } catch(NumberFormatException nfe) {
+ throw new IllegalArgumentException("The field expression '"+fieldExpression+"' is incorrect. Field number '"+fieldNumStr+" is not numeric", nfe);
+ }
+ if(pos < 0) {
+ throw new IllegalArgumentException("Negative position is not possible");
+ }
+ // pass down the remainder (after the dot) of the fieldExpression to the type at that position.
+ if(dotPos != -1) { // we need to go deeper
+ String rem = fieldExpression.substring(dotPos+1);
+ if( !(types[pos] instanceof CompositeType<?>) ) {
+ throw new RuntimeException("Element at position "+pos+" is not a composite type. There are no nested types to select");
+ }
+ CompositeType<?> cType = (CompositeType<?>) types[pos];
+ cType.getKey(rem, offset + pos, result);
+ return;
+ }
+
+ if(pos >= types.length) {
+ throw new IllegalArgumentException("The specified tuple position does not exist");
+ }
+
+ // count nested fields before "pos".
+ for(int i = 0; i < pos; i++) {
+ offset += types[i].getTotalFields() - 1; // this adds only something to offset if its a composite type.
+ }
+
+ result.add(new FlatFieldDescriptor(offset + pos, types[pos]));
+ }
+
public <X> TypeInformation<X> getTypeAt(int pos) {
if (pos < 0 || pos >= this.types.length) {
throw new IndexOutOfBoundsException();
@@ -115,4 +221,5 @@ public abstract class TupleTypeInfoBase<T> extends TypeInformation<T> implements
bld.append('>');
return bld.toString();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 55f6b1f..bc92cdd 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.avro.generic.GenericContainer;
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.CrossFunction;
@@ -47,13 +48,19 @@ import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.types.Value;
import org.apache.flink.util.Collector;
import org.apache.hadoop.io.Writable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
public class TypeExtractor {
+ private static final Logger LOG = LoggerFactory.getLogger(TypeExtractor.class);
// We need this to detect recursive types and not get caught
// in an endless recursion
@@ -220,6 +227,29 @@ public class TypeExtractor {
// get info from hierarchy
return (TypeInformation<OUT>) createTypeInfoWithTypeHierarchy(typeHierarchy, returnType, in1Type, in2Type);
}
+
+
+ /**
+ * @param curT : start type
+ * @return Type The immediate child of the top class
+ */
+ private Type recursivelyGetTypeHierarchy(ArrayList<Type> typeHierarchy, Type curT, Class<?> stopAtClass) {
+ while (!(curT instanceof ParameterizedType && ((Class<?>) ((ParameterizedType) curT).getRawType()).equals(
+ stopAtClass))
+ && !(curT instanceof Class<?> && ((Class<?>) curT).equals(stopAtClass))) {
+ typeHierarchy.add(curT);
+
+ // parameterized type
+ if (curT instanceof ParameterizedType) {
+ curT = ((Class<?>) ((ParameterizedType) curT).getRawType()).getGenericSuperclass();
+ }
+ // class
+ else {
+ curT = ((Class<?>) curT).getGenericSuperclass();
+ }
+ }
+ return curT;
+ }
@SuppressWarnings({ "unchecked", "rawtypes" })
private <IN1, IN2, OUT> TypeInformation<OUT> createTypeInfoWithTypeHierarchy(ArrayList<Type> typeHierarchy, Type t,
@@ -227,7 +257,7 @@ public class TypeExtractor {
// check if type is a subclass of tuple
if ((t instanceof Class<?> && Tuple.class.isAssignableFrom((Class<?>) t))
- || (t instanceof ParameterizedType && Tuple.class.isAssignableFrom((Class<?>) ((ParameterizedType) t).getRawType()))) {
+ || (t instanceof ParameterizedType && Tuple.class.isAssignableFrom((Class<?>) ((ParameterizedType) t).getRawType()))) {
Type curT = t;
@@ -236,7 +266,7 @@ public class TypeExtractor {
throw new InvalidTypesException(
"Usage of class Tuple as a type is not allowed. Use a concrete subclass (e.g. Tuple1, Tuple2, etc.) instead.");
}
-
+
// go up the hierarchy until we reach immediate child of Tuple (with or without generics)
// collect the types while moving up for a later top-down
while (!(curT instanceof ParameterizedType && ((Class<?>) ((ParameterizedType) curT).getRawType()).getSuperclass().equals(
@@ -295,15 +325,23 @@ public class TypeExtractor {
}
}
- // TODO: Check that type that extends Tuple does not have additional fields.
- // Right now, these fields are not be serialized by the TupleSerializer.
- // We might want to add an ExtendedTupleSerializer for that.
-
+ Class<?> tAsClass = null;
if (t instanceof Class<?>) {
- return new TupleTypeInfo(((Class<? extends Tuple>) t), tupleSubTypes);
+ tAsClass = (Class<?>) t;
} else if (t instanceof ParameterizedType) {
- return new TupleTypeInfo(((Class<? extends Tuple>) ((ParameterizedType) t).getRawType()), tupleSubTypes);
+ tAsClass = (Class<? extends Tuple>) ((ParameterizedType) t).getRawType();
}
+ Preconditions.checkNotNull(tAsClass, "t has a unexpected type");
+ // check if the class we assumed to be a Tuple so far is actually a pojo because it contains additional fields.
+ // check for additional fields.
+ int fieldCount = countFieldsInClass(tAsClass);
+ if(fieldCount != tupleSubTypes.length) {
+ // the class is not a real tuple because it contains additional fields. treat as a pojo
+ return (TypeInformation<OUT>) analyzePojo(tAsClass, new ArrayList<Type>() ); // the typeHierarchy here should be sufficient, even though it stops at the Tuple.class.
+ }
+
+ return new TupleTypeInfo(tAsClass, tupleSubTypes);
+
}
// type depends on another type
// e.g. class MyMapper<E> extends MapFunction<String, E>
@@ -361,16 +399,29 @@ public class TypeExtractor {
}
// objects with generics are treated as raw type
else if (t instanceof ParameterizedType) {
- return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType());
+ return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), new ArrayList<Type>()); // pass new type hierarchies here because
+ // while creating the TH here, we assumed a tuple type.
}
// no tuple, no TypeVariable, no generic type
else if (t instanceof Class) {
- return privateGetForClass((Class<OUT>) t);
+ return privateGetForClass((Class<OUT>) t, new ArrayList<Type>());
}
throw new InvalidTypesException("Type Information could not be created.");
}
+ private int countFieldsInClass(Class<?> clazz) {
+ int fieldCount = 0;
+ for(Field field : clazz.getFields()) { // get all fields
+ if( !Modifier.isStatic(field.getModifiers()) &&
+ !Modifier.isTransient(field.getModifiers())
+ ) {
+ fieldCount++;
+ }
+ }
+ return fieldCount;
+ }
+
private <IN1, IN2> TypeInformation<?> createTypeInfoFromInputs(TypeVariable<?> returnTypeVar, ArrayList<Type> returnTypeHierarchy,
TypeInformation<IN1> in1TypeInfo, TypeInformation<IN2> in2TypeInfo) {
@@ -383,7 +434,7 @@ public class TypeExtractor {
else {
returnTypeVar = (TypeVariable<?>) matReturnTypeVar;
}
-
+
TypeInformation<?> info = null;
if (in1TypeInfo != null) {
// find the deepest type variable that describes the type of input 1
@@ -806,11 +857,11 @@ public class TypeExtractor {
}
public static <X> TypeInformation<X> getForClass(Class<X> clazz) {
- return new TypeExtractor().privateGetForClass(clazz);
+ return new TypeExtractor().privateGetForClass(clazz, new ArrayList<Type>());
}
@SuppressWarnings("unchecked")
- private <X> TypeInformation<X> privateGetForClass(Class<X> clazz) {
+ private <X> TypeInformation<X> privateGetForClass(Class<X> clazz, ArrayList<Type> typeHierarchy) {
Validate.notNull(clazz);
// check for abstract classes or interfaces
@@ -819,10 +870,8 @@ public class TypeExtractor {
}
if (clazz.equals(Object.class)) {
- // this will occur when trying to analyze POJOs that have generic, this
- // exception will be caught and a GenericTypeInfo will be created for the type.
- // at some point we might support this using Kryo
- throw new InvalidTypesException("Object is not a valid type.");
+ // TODO (merging): better throw an exception here. the runtime does not support it yet
+ return new GenericTypeInfo<X>(clazz);
}
// check for arrays
@@ -879,62 +928,135 @@ public class TypeExtractor {
// special case handling for Class, this should not be handled by the POJO logic
return new GenericTypeInfo<X>(clazz);
}
+ if(GenericContainer.class.isAssignableFrom(clazz)) {
+ // this is a type generated by Avro. GenericTypeInfo is able to handle this case because its using Avro.
+ return new GenericTypeInfo<X>(clazz);
+ }
+ TypeInformation<X> pojoType = analyzePojo(clazz, typeHierarchy);
+ if (pojoType != null) {
+ return pojoType;
+ }
-// Disable POJO types for now (see https://mail-archives.apache.org/mod_mbox/incubator-flink-dev/201407.mbox/%3C53D96049.1060509%40cse.uta.edu%3E)
-//
-// TypeInformation<X> pojoType = analyzePojo(clazz);
-// if (pojoType != null) {
-// return pojoType;
-// }
// return a generic type
return new GenericTypeInfo<X>(clazz);
}
+
+ /**
+ * Checks if the given field is a valid pojo field:
+ * - it is public
+ * OR
+ * - there are getter and setter methods for the field.
+ *
+ * @param f field to check
+ * @param clazz class of field
+ * @param typeHierarchy type hierarchy for materializing generic types
+ * @return
+ */
+ private boolean isValidPojoField(Field f, Class<?> clazz, ArrayList<Type> typeHierarchy) {
+ if(Modifier.isPublic(f.getModifiers())) {
+ return true;
+ } else {
+ boolean hasGetter = false, hasSetter = false;
+ final String fieldNameLow = f.getName().toLowerCase();
+
+ Type fieldType = f.getGenericType();
+ TypeVariable<?> fieldTypeGeneric = null;
+ if(fieldType instanceof TypeVariable) {
+ fieldTypeGeneric = (TypeVariable<?>) fieldType;
+ fieldType = materializeTypeVariable(typeHierarchy, (TypeVariable<?>)fieldType);
+ }
+ for(Method m : clazz.getMethods()) {
+ // check for getter
+
+ if( // The name should be "get<FieldName>".
+ m.getName().toLowerCase().contains("get"+fieldNameLow) &&
+ // no arguments for the getter
+ m.getParameterTypes().length == 0 &&
+ // return type is same as field type (or the generic variant of it)
+ m.getReturnType().equals( fieldType ) || (fieldTypeGeneric != null && m.getGenericReturnType().equals(fieldTypeGeneric) )
+ ) {
+ if(hasGetter) {
+ throw new IllegalStateException("Detected more than one getters");
+ }
+ hasGetter = true;
+ }
+ // check for setters
+ if( m.getName().toLowerCase().contains("set"+fieldNameLow) &&
+ m.getParameterTypes().length == 1 && // one parameter of the field's type
+ ( m.getParameterTypes()[0].equals( fieldType ) || (fieldTypeGeneric != null && m.getGenericParameterTypes()[0].equals(fieldTypeGeneric) ) )&&
+ // return type is void.
+ m.getReturnType().equals(Void.TYPE)
+ ) {
+ if(hasSetter) {
+ throw new IllegalStateException("Detected more than one getters");
+ }
+ hasSetter = true;
+ }
+ }
+ if( hasGetter && hasSetter) {
+ return true;
+ } else {
+ if(!hasGetter) {
+ LOG.warn("Class "+clazz+" does not contain a getter for field "+f.getName() );
+ }
+ if(!hasSetter) {
+ LOG.warn("Class "+clazz+" does not contain a setter for field "+f.getName() );
+ }
+ return false;
+ }
+ }
+ }
- @SuppressWarnings("unused")
- private <X> TypeInformation<X> analyzePojo(Class<X> clazz) {
- List<Field> fields = getAllDeclaredFields(clazz);
+ private <X> TypeInformation<X> analyzePojo(Class<X> clazz, ArrayList<Type> typeHierarchy) {
+ // try to create Type hierarchy, if the incoming one is empty.
+ if(typeHierarchy.size() == 0) {
+ recursivelyGetTypeHierarchy(typeHierarchy, clazz, Object.class);
+ }
+
+ List<Field> fields = removeNonObjectFields(getAllDeclaredFields(clazz));
List<PojoField> pojoFields = new ArrayList<PojoField>();
for (Field field : fields) {
+ Type fieldType = field.getGenericType();
+ if(!isValidPojoField(field, clazz, typeHierarchy)) {
+ LOG.warn("Class "+clazz+" is not a valid POJO type");
+ return null;
+ }
try {
- if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) {
- pojoFields.add(new PojoField(field, privateCreateTypeInfo(field.getType())));
- }
+ typeHierarchy.add(fieldType);
+ pojoFields.add(new PojoField(field, createTypeInfoWithTypeHierarchy(typeHierarchy, fieldType, null, null) ));
} catch (InvalidTypesException e) {
- // If some of the fields cannot be analyzed, just return a generic type info
- // right now this happens when a field is an interface (collections are the prominent case here) or
- // when the POJO is generic, in which case the fields will have type Object.
- // We might fix that in the future when we use Kryo.
- return new GenericTypeInfo<X>(clazz);
+ //pojoFields.add(new PojoField(field, new GenericTypeInfo( Object.class ))); // we need kryo to properly serialize this
+ throw new InvalidTypesException("Flink is currently unable to serialize this type: "+fieldType+""
+ + "\nThe system is using the Avro serializer which is not able to handle all types.", e);
}
}
- PojoTypeInfo<X> pojoType = new PojoTypeInfo<X>(clazz, pojoFields);
+ CompositeType<X> pojoType = new PojoTypeInfo<X>(clazz, pojoFields);
+ //
+ // Validate the correctness of the pojo.
+ // returning "null" will result create a generic type information.
+ //
List<Method> methods = getAllDeclaredMethods(clazz);
- boolean containsReadObjectOrWriteObject = false;
for (Method method : methods) {
if (method.getName().equals("readObject") || method.getName().equals("writeObject")) {
- containsReadObjectOrWriteObject = true;
- break;
+ LOG.warn("Class "+clazz+" contains custom serialization methods we do not call.");
+ return null;
}
}
// Try retrieving the default constructor, if it does not have one
// we cannot use this because the serializer uses it.
- boolean hasDefaultCtor = true;
try {
clazz.getDeclaredConstructor();
} catch (NoSuchMethodException e) {
- hasDefaultCtor = false;
+ LOG.warn("Class "+clazz+" does not have a default constructor. You can not use it as a POJO");
+ return null;
}
-
-
- if (!containsReadObjectOrWriteObject && hasDefaultCtor) {
- return pojoType;
- }
-
- return null;
+
+ // everything is checked, we return the pojo
+ return pojoType;
}
// recursively determine all declared fields
@@ -950,6 +1072,19 @@ public class TypeExtractor {
return result;
}
+ /**
+ * Remove transient and static fields from a list of fields.
+ */
+ private static List<Field> removeNonObjectFields(List<Field> fields) {
+ List<Field> result = new ArrayList<Field>();
+ for(Field field: fields) {
+ if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) {
+ result.add(field);
+ }
+ }
+ return result;
+ }
+
// recursively determine all declared methods
private static List<Method> getAllDeclaredMethods(Class<?> clazz) {
List<Method> result = new ArrayList<Method>();
@@ -976,6 +1111,11 @@ public class TypeExtractor {
if (value instanceof Tuple) {
Tuple t = (Tuple) value;
int numFields = t.getArity();
+ if(numFields != countFieldsInClass(value.getClass())) {
+ // not a tuple since it has more fields.
+ return analyzePojo((Class<X>) value.getClass(), new ArrayList<Type>()); // we immediately call analyze Pojo here, because
+ // there is currently no other type that can handle such a class.
+ }
TypeInformation<?>[] infos = new TypeInformation[numFields];
for (int i = 0; i < numFields; i++) {
@@ -988,10 +1128,9 @@ public class TypeExtractor {
infos[i] = privateGetForObject(field);
}
-
return (TypeInformation<X>) new TupleTypeInfo(value.getClass(), infos);
} else {
- return privateGetForClass((Class<X>) value.getClass());
+ return privateGetForClass((Class<X>) value.getClass(), new ArrayList<Type>());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeInfoParser.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeInfoParser.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeInfoParser.java
index 6894e5a..953b69c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeInfoParser.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeInfoParser.java
@@ -87,7 +87,7 @@ public class TypeInfoParser {
}
return (TypeInformation<X>) parse(new StringBuilder(clearedString));
} catch (Exception e) {
- throw new IllegalArgumentException("String could not be parsed: " + e.getMessage());
+ throw new IllegalArgumentException("String could not be parsed: " + e.getMessage(), e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
index 5045b38..b3c25e4 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/ValueTypeInfo.java
@@ -53,6 +53,11 @@ public class ValueTypeInfo<T extends Value> extends TypeInformation<T> implement
}
@Override
+ public int getTotalFields() {
+ return 1;
+ }
+
+ @Override
public Class<T> getTypeClass() {
return this.type;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
index 19bcf0b..8c9e948 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/WritableTypeInfo.java
@@ -67,6 +67,11 @@ public class WritableTypeInfo<T extends Writable> extends TypeInformation<T> imp
public int getArity() {
return 1;
}
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
@Override
public Class<T> getTypeClass() {
@@ -88,6 +93,20 @@ public class WritableTypeInfo<T extends Writable> extends TypeInformation<T> imp
return "WritableType<" + typeClass.getName() + ">";
}
+ @Override
+ public int hashCode() {
+ return typeClass.hashCode() ^ 0xd3a2646c;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() == WritableTypeInfo.class) {
+ return typeClass == ((WritableTypeInfo<?>) obj).typeClass;
+ } else {
+ return false;
+ }
+ }
+
// --------------------------------------------------------------------------------------------
static final <T extends Writable> TypeInformation<T> getWritableTypeInfo(Class<T> typeClass) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueComparator.java
index 0b7890f..9b3b191 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/CopyableValueComparator.java
@@ -43,8 +43,6 @@ public class CopyableValueComparator<T extends CopyableValue<T> & Comparable<T>>
private transient T tempReference;
- private final Comparable<?>[] extractedKey = new Comparable[1];
-
private final TypeComparator<?>[] comparators = new TypeComparator[] {this};
public CopyableValueComparator(boolean ascending, Class<T> type) {
@@ -126,13 +124,13 @@ public class CopyableValueComparator<T extends CopyableValue<T> & Comparable<T>>
}
@Override
- public Object[] extractKeys(T record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
@Override
- public TypeComparator<?>[] getComparators() {
+ public TypeComparator<?>[] getFlatComparators() {
return comparators;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/926f835a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/GenericTypeComparator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/GenericTypeComparator.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/GenericTypeComparator.java
index 66cbdf4..2d3ce39 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/GenericTypeComparator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/runtime/GenericTypeComparator.java
@@ -53,9 +53,6 @@ public class GenericTypeComparator<T extends Comparable<T>> extends TypeComparat
private transient Kryo kryo;
@SuppressWarnings("rawtypes")
- private final Comparable[] extractedKey = new Comparable[1];
-
- @SuppressWarnings("rawtypes")
private final TypeComparator[] comparators = new TypeComparator[] {this};
// ------------------------------------------------------------------------
@@ -171,14 +168,14 @@ public class GenericTypeComparator<T extends Comparable<T>> extends TypeComparat
}
@Override
- public Object[] extractKeys(T record) {
- extractedKey[0] = record;
- return extractedKey;
+ public int extractKeys(Object record, Object[] target, int index) {
+ target[index] = record;
+ return 1;
}
- @Override
@SuppressWarnings("rawtypes")
- public TypeComparator[] getComparators() {
+ @Override
+ public TypeComparator[] getFlatComparators() {
return comparators;
}
[10/11] git commit: Documentation and fixes
Posted by rm...@apache.org.
Documentation and fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/598ae376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/598ae376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/598ae376
Branch: refs/heads/master
Commit: 598ae3765cc45c2ede00fd4e32adf4a4a3327615
Parents: fd0be2f
Author: Robert Metzger <rm...@apache.org>
Authored: Tue Oct 7 15:44:38 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
docs/dataset_transformations.md | 56 +--
docs/programming_guide.md | 92 +++-
.../api/java/typeutils/GenericTypeInfo.java | 10 +
.../flink/api/java/typeutils/TypeExtractor.java | 3 +-
.../javaApiOperators/GroupReduceITCase.java | 26 +-
.../util/CollectionDataSets.java | 448 ++++++++++---------
6 files changed, 372 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/docs/dataset_transformations.md
----------------------------------------------------------------------
diff --git a/docs/dataset_transformations.md b/docs/dataset_transformations.md
index f968b1d..78a3fd5 100644
--- a/docs/dataset_transformations.md
+++ b/docs/dataset_transformations.md
@@ -176,11 +176,12 @@ DataSet<Tuple2<String, Integer>> out = in.project(2,0).types(String.class, Integ
### Transformations on Grouped DataSet
The reduce operations can operate on grouped data sets. Specifying the key to
-be used for grouping can be done in two ways:
+be used for grouping can be done in many ways:
-- a key-selector function or
-- one or more field position keys (Tuple DataSet only).
-- Case Class fields (Case Classes only).
+- key expressions
+- a key-selector function
+- one or more field position keys (Tuple DataSet only)
+- Case Class fields (Case Classes only)
Please look at the reduce examples to see how the grouping keys are specified.
@@ -220,11 +221,8 @@ public class WordCounter implements ReduceFunction<WC> {
// [...]
DataSet<WC> words = // [...]
DataSet<WC> wordCounts = words
- // DataSet grouping with inline-defined KeySelector function
- .groupBy(
- new KeySelector<WC, String>() {
- public String getKey(WC wc) { return wc.word; }
- })
+ // DataSet grouping on field "word"
+ .groupBy("word")
// apply ReduceFunction on grouped DataSet
.reduce(new WordCounter());
~~~
@@ -360,11 +358,10 @@ Works analogous to grouping by Case Class fields in *Reduce* transformations.
Works analogous to key-selector functions in *Reduce* transformations.
-#### GroupReduce on sorted groups (Tuple DataSets only)
+#### GroupReduce on sorted groups
A group-reduce function accesses the elements of a group using an Iterable. Optionally, the Iterable can hand out the elements of a group in a specified order. In many cases this can help to reduce the complexity of a user-defined
group-reduce function and improve its efficiency.
-Right now, this feature is only available for DataSets of Tuples.
The following code shows another example how to remove duplicate Strings in a DataSet grouped by an Integer and sorted by String.
@@ -656,7 +653,8 @@ val output = input.aggregate(SUM, 0).and(MIN, 2)
The Join transformation joins two DataSets into one DataSet. The elements of both DataSets are joined on one or more keys which can be specified using
-- a key-selector function or
+- a kex expression
+- a key-selector function
- one or more field position keys (Tuple DataSet only).
- Case Class Fields
@@ -672,13 +670,15 @@ The following code shows a default Join transformation using field position keys
<div data-lang="java" markdown="1">
~~~java
-DataSet<Tuple2<Integer, String>> input1 = // [...]
-DataSet<Tuple2<Double, Integer>> input2 = // [...]
+public static class User { public String name; public int zip; }
+public static class Store { public Manager mgr; public int zip; }
+DataSet<User> input1 = // [...]
+DataSet<Store> input2 = // [...]
// result dataset is typed as Tuple2
-DataSet<Tuple2<Tuple2<Integer, String>, Tuple2<Double, Integer>>>
+DataSet<Tuple2<User, Store>>
result = input1.join(input2)
- .where(0) // key of the first input
- .equalTo(1); // key of the second input
+ .where("zip") // key of the first input (users)
+ .equalTo("zip"); // key of the second input (stores)
~~~
</div>
@@ -729,14 +729,10 @@ DataSet<Tuple2<String, Double>>
ratings.join(weights)
// key of the first input
- .where(new KeySelection<Rating, String>() {
- public String getKey(Rating r) { return r.category; }
- })
+ .where("category)
// key of the second input
- .equalTo(new KeySelection<Tuple2<String, Double>, String>() {
- public String getKey(Tuple2<String, Double> t) { return t.f0; }
- })
+ .equalTo("f0")
// applying the JoinFunction on joining pairs
.with(new PointWeighter());
@@ -1000,17 +996,14 @@ val result1 = input1.crossWithHuge(input2)
The CoGroup transformation jointly processes groups of two DataSets. Both DataSets are grouped on a defined key and groups of both DataSets that share the same key are handed together to a user-defined co-group function. If for a specific key only one DataSet has a group, the co-group function is called with this group and an empty group.
A co-group function can separately iterate over the elements of both groups and return an arbitrary number of result elements.
-Similar to Reduce, GroupReduce, and Join, keys can be defined using
+Similar to Reduce, GroupReduce, and Join, keys can be defined using the different key-selection methods.
-- a key-selector function or
-- one or more field position keys (Tuple DataSet only) or
-- Case Class fields.
-
-#### CoGroup on DataSets Grouped by Field Position Keys (Tuple DataSets only)
+#### CoGroup on DataSets
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
+The example shows how to group by Field Position Keys (Tuple DataSets only). You can do the same with Pojo-types and key expressions.
~~~java
// Some CoGroupFunction definition
class MyCoGrouper
@@ -1071,9 +1064,6 @@ val output = iVals.coGroup(dVals).where(0).equalTo(0) {
</div>
</div>
-#### CoGroup on DataSets Grouped by Key-Selector Function
-
-Works analogous to key-selector functions in Join transformations.
### Union
@@ -1132,7 +1122,7 @@ val out = in.rebalance().map { ... }
### Hash-Partition
Hash-partitions a DataSet on a given key.
-Keys can be specified as key-selector functions or field position keys (see [Reduce examples](#reduce-on-grouped-dataset) for how to specify keys).
+Keys can be specified as key expressions or field position keys (see [Reduce examples](#reduce-on-grouped-dataset) for how to specify keys).
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/docs/programming_guide.md
----------------------------------------------------------------------
diff --git a/docs/programming_guide.md b/docs/programming_guide.md
index 8d0b299..fdea599 100644
--- a/docs/programming_guide.md
+++ b/docs/programming_guide.md
@@ -871,6 +871,8 @@ you do not need to physically pack the data set types into keys and
values. Keys are "virtual": they are defined as functions over the
actual data to guide the grouping operator.
+### Define keys for Tuples
+
The simplest case is grouping a data set of Tuples on one or more
fields of the Tuple:
{% highlight java %}
@@ -895,7 +897,75 @@ The data set is grouped on the composite key consisting of the first and the
second fields, therefore the GroupReduceFuntion will receive groups
with the same value for both fields.
-In general, key definition is done via a "key selector" function, which
+A note on nested Tuples: If you have a DataSet with a nested tuple, such as:
+{% highlight java %}
+DataSet<Tuple3<Tuple2<Integer, Float>,String,Long>> ds;
+{% endhighlight %}
+Specifying `groupBy(0)` will cause the system to use the full `Tuple2` as a key (with the Integer and Float being the key). If you want to "navigate" into the nested `Tuple2`, you have to use a string-based expression, as explained below. For this particular example, you would have to specfiy `f0.f0`.
+
+### Define key using a String Expression
+Starting from release 0.7-incubating, you can use String-based key expressions to select keys.
+
+The String expressions allow to specify the name of the field in a class you want to group by.
+
+In the example below, we have a `WC` POJO with two fields "word" and "count". To group by the field "word", we just pass this name to the `groupBy()` function.
+{% highlight java %}
+// some ordinary POJO (Plain old Java Object)
+public class WC {
+ public String word;
+ public int count;
+}
+DataSet<WC> words = // [...]
+DataSet<WC> wordCounts = words.groupBy("word").reduce(/*do something*/);
+{% endhighlight %}
+
+**Conditions** for a class to be treated as a POJO by Flink:
+
+- The class must be public
+- It must have a public constructor without arguments
+- All fields either have to be public or there must be getters and setters for all non-public fields. If the field name is `foo` the getter and setters must be called `getFoo()` and `setFoo()`.
+
+**Valid Expressions**:
+
+- You can select POJO fields by their field name
+- You can select Tuple fields by their field name as well. For example `f0` or `f5`.
+- You can select nested fields in POJOs and Tuples. Expressions like `user.zip` or `user.groupId` are valid. Flink also supports POJOs inside Tuples: `f1.user.zip`.
+- You can select all fields at each level. To select all fields, specify `*`. This also works for the nested case: `user.*`.
+
+**Example for nested POJO**
+
+{% highlight java %}
+public static class WC {
+ public ComplexNestedClass complex; //nested POJO
+ private int count;
+ // getter / setter for private field (count)
+ public int getCount() {
+ return count;
+ }
+ public void setCount(int c) {
+ this.count = c;
+ }
+}
+public static class ComplexNestedClass {
+ public Integer someNumber;
+ public float someFloat;
+ public Tuple3<Long, Long, String> word;
+ public IntWritable hadoopCitizen;
+}
+{% endhighlight %}
+
+These are valid expressions for the example POJO above:
+
+- `count`: The count field in the `WC` class.
+- `complex.*`: Selects all fields in the `ComplexNestedClass`.
+- `complex.word.f2`: Selects the last field in the Tuple3.
+- `complex.hadoopCitizen`: Selects a Hadoop-`Writable` type as a key.
+
+Please note that you can only use types inside POJOs that Flink is able to serialize. Currently, we are using [Avro](http://avro.apache.org) to serialize arbitrary objects (such as `Date`).
+
+### Define key using a Key Selector Function
+
+An additional way to define keys are "key selector" functions, which
takes as argument one dataset element and returns a key of an
arbitrary data type by performing an arbitrary computation on this
element. For example:
@@ -1053,8 +1123,7 @@ data.map(new MapFunction<String, Integer> () {
#### Java 8 Lambdas
-***Warning: Lambdas are currently only supported for filter and reduce
- transformations***
+Flink also supports Java 8 Lambdas in the Java API. Please see the full [Java 8 Guide](java8_programming_guide.html).
{% highlight java %}
DataSet<String> data = // [...]
@@ -1179,7 +1248,7 @@ efficient execution strategies.
There are four different categories of data types, which are treated slightly different when it
to [specifying keys](#specifying-keys):
-1. **General Types**
+1. **General Types and POJOs**
2. **Tuples**/**Case Classes**
3. **Values**
4. **Hadoop Writables**
@@ -1276,17 +1345,13 @@ wordCounts.map { _.count }
When working with operators that require a Key for grouping or matching records
-you need to implement a key selector function for your custom type (see
+you can select the key using a key expression (see
[Specifying Keys](#specifying-keys)).
<div class="codetabs" markdown="1">
<div data-lang="java" markdown="1">
{% highlight java %}
-wordCounts.groupBy(new KeySelector<WordCount, String>() {
- public String getKey(WordCount v) {
- return v.word;
- }
-}).reduce(new MyReduceFunction());
+wordCounts.groupBy("word").reduce(new MyReduceFunction());
{% endhighlight %}
</div>
<div data-lang="scala" markdown="1">
@@ -1328,16 +1393,19 @@ than one position to use composite keys (see [Section Data Transformations](#tra
{% highlight java %}
wordCounts
- .groupBy(0)
+ .groupBy(0) // also valid .groupBy("f0")
.reduce(new MyReduceFunction());
{% endhighlight %}
+Also, you can "navigate" into nested tuples using (String) key expressions.
In order to access fields more intuitively and to generate more readable code, it is also possible
to extend a subclass of Tuple. You can add getters and setters with custom names that delegate to
the field positions. See this
-{% gh_link /flink-examples/flink-java-examples/src/main/java/org/apache/flink/example/java/relational/TPCHQuery3.java "example" %} for an
+{% gh_link /flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/TPCHQuery3.java "example" %} for an
illustration how to make use of that mechanism.
+Note that if you are extending from a Tuple and add fields to your class, it will be treated as a POJO.
+
</div>
<div data-lang="scala" markdown="1">
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
index 6272538..7b0c5bf 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/GenericTypeInfo.java
@@ -25,6 +25,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.typeutils.runtime.AvroSerializer;
import org.apache.flink.api.java.typeutils.runtime.GenericTypeComparator;
+import java.util.Collection;
+
/**
*
@@ -32,9 +34,17 @@ import org.apache.flink.api.java.typeutils.runtime.GenericTypeComparator;
public class GenericTypeInfo<T> extends TypeInformation<T> implements AtomicType<T> {
private final Class<T> typeClass;
+ private final static Class[] unsupportedByAvro = new Class[] {Collection.class};
public GenericTypeInfo(Class<T> typeClass) {
this.typeClass = typeClass;
+ for(Class unsupported: unsupportedByAvro) {
+ if(unsupported.isAssignableFrom(typeClass)) {
+ throw new RuntimeException("The type '"+typeClass+"' is currently not supported " +
+ "by the Avro Serializer that Flink is using for serializing " +
+ "arbitrary objects");
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 5935e31..7836e74 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -1065,7 +1065,8 @@ public class TypeExtractor {
/**
* recursively determine all declared fields
- * This is required because getFields() is not returning
+ * This is required because class.getFields() is not returning fields defined
+ * in parent classes.
*/
public static List<Field> getAllDeclaredFields(Class<?> clazz) {
List<Field> result = new ArrayList<Field>();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index 63c4dfc..cb79688 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -636,9 +636,9 @@ public class GroupReduceITCase extends JavaProgramTestBase {
env.execute();
// return expected result
- return "a--(1,1)-(1,2)-(1,3)-\n" +
+ return "a--(2,1)-(1,3)-(1,2)-\n" +
"b--(2,2)-\n"+
- "c--(3,3)-(3,6)-(3,9)-\n";
+ "c--(4,9)-(3,6)-(3,3)-\n";
}
case 22: {
/*
@@ -737,24 +737,17 @@ public class GroupReduceITCase extends JavaProgramTestBase {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setDegreeOfParallelism(1);
- DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getPojoWithMultiplePojos(env);
+ DataSet<CollectionDataSets.PojoWithMultiplePojos> ds = CollectionDataSets.getPojoWithMultiplePojos(env);
// f0.f0 is first integer
- DataSet<String> reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING)
- .reduceGroup(new GroupReduceFunction<CollectionDataSets.PojoContainingTupleAndWritable, String>() {
+ DataSet<String> reduceDs = ds.groupBy("p2.a2")
+ .reduceGroup(new GroupReduceFunction<CollectionDataSets.PojoWithMultiplePojos, String>() {
@Override
public void reduce(
- Iterable<PojoContainingTupleAndWritable> values,
+ Iterable<CollectionDataSets.PojoWithMultiplePojos> values,
Collector<String> out) throws Exception {
- boolean once = false;
StringBuilder concat = new StringBuilder();
- for(PojoContainingTupleAndWritable value : values) {
- if(!once) {
- concat.append(value.hadoopFan.get());
- concat.append("---");
- once = true;
- }
- concat.append(value.theTuple);
- concat.append("-");
+ for(CollectionDataSets.PojoWithMultiplePojos value : values) {
+ concat.append(value.p2.a2);
}
out.collect(concat.toString());
}
@@ -763,8 +756,7 @@ public class GroupReduceITCase extends JavaProgramTestBase {
env.execute();
// return expected result
- return "1---(10,100)-\n" +
- "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n";
+ return "b\nccc\nee\n";
}
default: {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/598ae376/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
index e5ab29a..6757f66 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
@@ -21,6 +21,7 @@ package org.apache.flink.test.javaApiOperators.util;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
@@ -44,139 +45,139 @@ import org.apache.hadoop.io.IntWritable;
public class CollectionDataSets {
public static DataSet<Tuple3<Integer, Long, String>> get3TupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple3<Integer, Long, String>> data = new ArrayList<Tuple3<Integer, Long, String>>();
- data.add(new Tuple3<Integer, Long, String>(1,1l,"Hi"));
- data.add(new Tuple3<Integer, Long, String>(2,2l,"Hello"));
- data.add(new Tuple3<Integer, Long, String>(3,2l,"Hello world"));
- data.add(new Tuple3<Integer, Long, String>(4,3l,"Hello world, how are you?"));
- data.add(new Tuple3<Integer, Long, String>(5,3l,"I am fine."));
- data.add(new Tuple3<Integer, Long, String>(6,3l,"Luke Skywalker"));
- data.add(new Tuple3<Integer, Long, String>(7,4l,"Comment#1"));
- data.add(new Tuple3<Integer, Long, String>(8,4l,"Comment#2"));
- data.add(new Tuple3<Integer, Long, String>(9,4l,"Comment#3"));
- data.add(new Tuple3<Integer, Long, String>(10,4l,"Comment#4"));
- data.add(new Tuple3<Integer, Long, String>(11,5l,"Comment#5"));
- data.add(new Tuple3<Integer, Long, String>(12,5l,"Comment#6"));
- data.add(new Tuple3<Integer, Long, String>(13,5l,"Comment#7"));
- data.add(new Tuple3<Integer, Long, String>(14,5l,"Comment#8"));
- data.add(new Tuple3<Integer, Long, String>(15,5l,"Comment#9"));
- data.add(new Tuple3<Integer, Long, String>(16,6l,"Comment#10"));
- data.add(new Tuple3<Integer, Long, String>(17,6l,"Comment#11"));
- data.add(new Tuple3<Integer, Long, String>(18,6l,"Comment#12"));
- data.add(new Tuple3<Integer, Long, String>(19,6l,"Comment#13"));
- data.add(new Tuple3<Integer, Long, String>(20,6l,"Comment#14"));
- data.add(new Tuple3<Integer, Long, String>(21,6l,"Comment#15"));
-
+ data.add(new Tuple3<Integer, Long, String>(1, 1l, "Hi"));
+ data.add(new Tuple3<Integer, Long, String>(2, 2l, "Hello"));
+ data.add(new Tuple3<Integer, Long, String>(3, 2l, "Hello world"));
+ data.add(new Tuple3<Integer, Long, String>(4, 3l, "Hello world, how are you?"));
+ data.add(new Tuple3<Integer, Long, String>(5, 3l, "I am fine."));
+ data.add(new Tuple3<Integer, Long, String>(6, 3l, "Luke Skywalker"));
+ data.add(new Tuple3<Integer, Long, String>(7, 4l, "Comment#1"));
+ data.add(new Tuple3<Integer, Long, String>(8, 4l, "Comment#2"));
+ data.add(new Tuple3<Integer, Long, String>(9, 4l, "Comment#3"));
+ data.add(new Tuple3<Integer, Long, String>(10, 4l, "Comment#4"));
+ data.add(new Tuple3<Integer, Long, String>(11, 5l, "Comment#5"));
+ data.add(new Tuple3<Integer, Long, String>(12, 5l, "Comment#6"));
+ data.add(new Tuple3<Integer, Long, String>(13, 5l, "Comment#7"));
+ data.add(new Tuple3<Integer, Long, String>(14, 5l, "Comment#8"));
+ data.add(new Tuple3<Integer, Long, String>(15, 5l, "Comment#9"));
+ data.add(new Tuple3<Integer, Long, String>(16, 6l, "Comment#10"));
+ data.add(new Tuple3<Integer, Long, String>(17, 6l, "Comment#11"));
+ data.add(new Tuple3<Integer, Long, String>(18, 6l, "Comment#12"));
+ data.add(new Tuple3<Integer, Long, String>(19, 6l, "Comment#13"));
+ data.add(new Tuple3<Integer, Long, String>(20, 6l, "Comment#14"));
+ data.add(new Tuple3<Integer, Long, String>(21, 6l, "Comment#15"));
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
}
-
+
public static DataSet<Tuple3<Integer, Long, String>> getSmall3TupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple3<Integer, Long, String>> data = new ArrayList<Tuple3<Integer, Long, String>>();
- data.add(new Tuple3<Integer, Long, String>(1,1l,"Hi"));
- data.add(new Tuple3<Integer, Long, String>(2,2l,"Hello"));
- data.add(new Tuple3<Integer, Long, String>(3,2l,"Hello world"));
-
+ data.add(new Tuple3<Integer, Long, String>(1, 1l, "Hi"));
+ data.add(new Tuple3<Integer, Long, String>(2, 2l, "Hello"));
+ data.add(new Tuple3<Integer, Long, String>(3, 2l, "Hello world"));
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
}
-
+
public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> get5TupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<Tuple5<Integer, Long, Integer, String, Long>>();
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(1,1l,0,"Hallo",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,2l,1,"Hallo Welt",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,3l,2,"Hallo Welt wie",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,4l,3,"Hallo Welt wie gehts?",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,5l,4,"ABC",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(3,6l,5,"BCD",3l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,7l,6,"CDE",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,8l,7,"DEF",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,9l,8,"EFG",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(4,10l,9,"FGH",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,11l,10,"GHI",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,12l,11,"HIJ",3l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,13l,12,"IJK",3l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,14l,13,"JKL",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(5,15l,14,"KLM",2l));
-
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(1, 1l, 0, "Hallo", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2, 2l, 1, "Hallo Welt", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2, 3l, 2, "Hallo Welt wie", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3, 4l, 3, "Hallo Welt wie gehts?", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3, 5l, 4, "ABC", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(3, 6l, 5, "BCD", 3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4, 7l, 6, "CDE", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4, 8l, 7, "DEF", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4, 9l, 8, "EFG", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(4, 10l, 9, "FGH", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5, 11l, 10, "GHI", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5, 12l, 11, "HIJ", 3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5, 13l, 12, "IJK", 3l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5, 14l, 13, "JKL", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(5, 15l, 14, "KLM", 2l));
+
Collections.shuffle(data);
-
- TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new
- TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>(
- BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.LONG_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.STRING_TYPE_INFO,
- BasicTypeInfo.LONG_TYPE_INFO
- );
-
+
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO
+ );
+
return env.fromCollection(data, type);
}
-
+
public static DataSet<Tuple5<Integer, Long, Integer, String, Long>> getSmall5TupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple5<Integer, Long, Integer, String, Long>> data = new ArrayList<Tuple5<Integer, Long, Integer, String, Long>>();
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(1,1l,0,"Hallo",1l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,2l,1,"Hallo Welt",2l));
- data.add(new Tuple5<Integer, Long, Integer, String, Long>(2,3l,2,"Hallo Welt wie",1l));
-
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(1, 1l, 0, "Hallo", 1l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2, 2l, 1, "Hallo Welt", 2l));
+ data.add(new Tuple5<Integer, Long, Integer, String, Long>(2, 3l, 2, "Hallo Welt wie", 1l));
+
Collections.shuffle(data);
-
- TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new
- TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>(
- BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.LONG_TYPE_INFO,
- BasicTypeInfo.INT_TYPE_INFO,
- BasicTypeInfo.STRING_TYPE_INFO,
- BasicTypeInfo.LONG_TYPE_INFO
- );
-
+
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>> type = new
+ TupleTypeInfo<Tuple5<Integer, Long, Integer, String, Long>>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO
+ );
+
return env.fromCollection(data, type);
}
-
+
public static DataSet<Tuple2<Tuple2<Integer, Integer>, String>> getSmallNestedTupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple2<Tuple2<Integer, Integer>, String>> data = new ArrayList<Tuple2<Tuple2<Integer, Integer>, String>>();
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,1), "one"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,2), "two"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,3), "three"));
-
- TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(1, 1), "one"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(2, 2), "two"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(3, 3), "three"));
+
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>>(
- new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO,BasicTypeInfo.INT_TYPE_INFO),
- BasicTypeInfo.STRING_TYPE_INFO
- );
-
+ new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
return env.fromCollection(data, type);
}
-
+
public static DataSet<Tuple2<Tuple2<Integer, Integer>, String>> getGroupSortedNestedTupleDataSet(ExecutionEnvironment env) {
-
+
List<Tuple2<Tuple2<Integer, Integer>, String>> data = new ArrayList<Tuple2<Tuple2<Integer, Integer>, String>>();
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,3), "a"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,2), "a"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,1), "a"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,2), "b"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,3), "c"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,6), "c"));
- data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(4,9), "c"));
-
- TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(1, 3), "a"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(1, 2), "a"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(2, 1), "a"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(2, 2), "b"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(3, 3), "c"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(3, 6), "c"));
+ data.add(new Tuple2<Tuple2<Integer, Integer>, String>(new Tuple2<Integer, Integer>(4, 9), "c"));
+
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>>(
- new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO,BasicTypeInfo.INT_TYPE_INFO),
- BasicTypeInfo.STRING_TYPE_INFO
- );
-
+ new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
return env.fromCollection(data, type);
}
-
+
public static DataSet<String> getStringDataSet(ExecutionEnvironment env) {
-
+
List<String> data = new ArrayList<String>();
data.add("Hi");
data.add("Hello");
@@ -186,14 +187,14 @@ public class CollectionDataSets {
data.add("Luke Skywalker");
data.add("Random comment");
data.add("LOL");
-
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
}
-
+
public static DataSet<Integer> getIntegerDataSet(ExecutionEnvironment env) {
-
+
List<Integer> data = new ArrayList<Integer>();
data.add(1);
data.add(2);
@@ -210,135 +211,146 @@ public class CollectionDataSets {
data.add(5);
data.add(5);
data.add(5);
-
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
}
-
+
public static DataSet<CustomType> getCustomTypeDataSet(ExecutionEnvironment env) {
-
+
List<CustomType> data = new ArrayList<CustomType>();
- data.add(new CustomType(1,0l,"Hi"));
- data.add(new CustomType(2,1l,"Hello"));
- data.add(new CustomType(2,2l,"Hello world"));
- data.add(new CustomType(3,3l,"Hello world, how are you?"));
- data.add(new CustomType(3,4l,"I am fine."));
- data.add(new CustomType(3,5l,"Luke Skywalker"));
- data.add(new CustomType(4,6l,"Comment#1"));
- data.add(new CustomType(4,7l,"Comment#2"));
- data.add(new CustomType(4,8l,"Comment#3"));
- data.add(new CustomType(4,9l,"Comment#4"));
- data.add(new CustomType(5,10l,"Comment#5"));
- data.add(new CustomType(5,11l,"Comment#6"));
- data.add(new CustomType(5,12l,"Comment#7"));
- data.add(new CustomType(5,13l,"Comment#8"));
- data.add(new CustomType(5,14l,"Comment#9"));
- data.add(new CustomType(6,15l,"Comment#10"));
- data.add(new CustomType(6,16l,"Comment#11"));
- data.add(new CustomType(6,17l,"Comment#12"));
- data.add(new CustomType(6,18l,"Comment#13"));
- data.add(new CustomType(6,19l,"Comment#14"));
- data.add(new CustomType(6,20l,"Comment#15"));
-
+ data.add(new CustomType(1, 0l, "Hi"));
+ data.add(new CustomType(2, 1l, "Hello"));
+ data.add(new CustomType(2, 2l, "Hello world"));
+ data.add(new CustomType(3, 3l, "Hello world, how are you?"));
+ data.add(new CustomType(3, 4l, "I am fine."));
+ data.add(new CustomType(3, 5l, "Luke Skywalker"));
+ data.add(new CustomType(4, 6l, "Comment#1"));
+ data.add(new CustomType(4, 7l, "Comment#2"));
+ data.add(new CustomType(4, 8l, "Comment#3"));
+ data.add(new CustomType(4, 9l, "Comment#4"));
+ data.add(new CustomType(5, 10l, "Comment#5"));
+ data.add(new CustomType(5, 11l, "Comment#6"));
+ data.add(new CustomType(5, 12l, "Comment#7"));
+ data.add(new CustomType(5, 13l, "Comment#8"));
+ data.add(new CustomType(5, 14l, "Comment#9"));
+ data.add(new CustomType(6, 15l, "Comment#10"));
+ data.add(new CustomType(6, 16l, "Comment#11"));
+ data.add(new CustomType(6, 17l, "Comment#12"));
+ data.add(new CustomType(6, 18l, "Comment#13"));
+ data.add(new CustomType(6, 19l, "Comment#14"));
+ data.add(new CustomType(6, 20l, "Comment#15"));
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
-
+
}
-
+
public static DataSet<CustomType> getSmallCustomTypeDataSet(ExecutionEnvironment env) {
-
+
List<CustomType> data = new ArrayList<CustomType>();
- data.add(new CustomType(1,0l,"Hi"));
- data.add(new CustomType(2,1l,"Hello"));
- data.add(new CustomType(2,2l,"Hello world"));
-
+ data.add(new CustomType(1, 0l, "Hi"));
+ data.add(new CustomType(2, 1l, "Hello"));
+ data.add(new CustomType(2, 2l, "Hello world"));
+
Collections.shuffle(data);
-
+
return env.fromCollection(data);
-
+
}
-
+
public static class CustomType implements Serializable {
-
+
private static final long serialVersionUID = 1L;
-
+
public int myInt;
public long myLong;
public String myString;
-
- public CustomType() {};
-
+
+ public CustomType() {
+ }
+
+ ;
+
public CustomType(int i, long l, String s) {
myInt = i;
myLong = l;
myString = s;
}
-
+
@Override
public String toString() {
- return myInt+","+myLong+","+myString;
+ return myInt + "," + myLong + "," + myString;
}
}
-
+
public static DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> getSmallTuplebasedPojoMatchingDataSet(ExecutionEnvironment env) {
List<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> data = new ArrayList<Tuple7<Integer, String, Integer, Integer, Long, String, Long>>();
- data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(2, "Second",20, 200, 2000L, "Two", 20000L));
- data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(3, "Third",30, 300, 3000L, "Three", 30000L));
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(2, "Second", 20, 200, 2000L, "Two", 20000L));
+ data.add(new Tuple7<Integer, String, Integer, Integer, Long, String, Long>(3, "Third", 30, 300, 3000L, "Three", 30000L));
return env.fromCollection(data);
}
-
+
public static DataSet<POJO> getSmallPojoDataSet(ExecutionEnvironment env) {
List<POJO> data = new ArrayList<POJO>();
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new POJO(2, "Second",20, 200, 2000L, "Two", 20000L));
- data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L));
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L));
+ data.add(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L));
return env.fromCollection(data);
}
-
+
public static DataSet<POJO> getDuplicatePojoDataSet(ExecutionEnvironment env) {
List<POJO> data = new ArrayList<POJO>();
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L)); // 5x
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new POJO(1, "First",10, 100, 1000L, "One", 10000L));
- data.add(new POJO(2, "Second",20, 200, 2000L, "Two", 20000L));
- data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L)); // 2x
- data.add(new POJO(3, "Third",30, 300, 3000L, "Three", 30000L));
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L)); // 5x
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(1, "First", 10, 100, 1000L, "One", 10000L));
+ data.add(new POJO(2, "Second", 20, 200, 2000L, "Two", 20000L));
+ data.add(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L)); // 2x
+ data.add(new POJO(3, "Third", 30, 300, 3000L, "Three", 30000L));
return env.fromCollection(data);
}
-
+
public static class POJO {
public int number;
public String str;
public Tuple2<Integer, CustomType> nestedTupleWithCustom;
public NestedPojo nestedPojo;
+ public Date date;
public transient Long ignoreMe;
- public POJO(int i0, String s0,
- int i1, int i2, long l0, String s1,
- long l1) {
+
+ public POJO(int i0, String s0,
+ int i1, int i2, long l0, String s1,
+ long l1) {
this.number = i0;
this.str = s0;
this.nestedTupleWithCustom = new Tuple2<Integer, CustomType>(i1, new CustomType(i2, l0, s1));
this.nestedPojo = new NestedPojo();
+ this.date = new Date();
this.nestedPojo.longNumber = l1;
}
- public POJO() {}
+
+ public POJO() {
+ }
+
@Override
public String toString() {
- return number+" "+str+" "+nestedTupleWithCustom+" "+nestedPojo.longNumber;
+ return number + " " + str + " " + nestedTupleWithCustom + " " + nestedPojo.longNumber;
}
}
-
+
public static class NestedPojo {
public static Object ignoreMe;
public long longNumber;
- public NestedPojo() {}
+
+ public NestedPojo() {
+ }
}
-
+
public static DataSet<CrazyNested> getCrazyNestedDataSet(ExecutionEnvironment env) {
List<CrazyNested> data = new ArrayList<CrazyNested>();
data.add(new CrazyNested("aa"));
@@ -349,16 +361,20 @@ public class CollectionDataSets {
data.add(new CrazyNested("cc"));
return env.fromCollection(data);
}
-
+
public static class CrazyNested {
public CrazyNestedL1 nest_Lvl1;
public Long something; // test proper null-value handling
- public CrazyNested() {}
+
+ public CrazyNested() {
+ }
+
public CrazyNested(String set, String second, long s) { // additional CTor to set all fields to non-null values
this(set);
something = s;
nest_Lvl1.a = second;
}
+
public CrazyNested(String set) {
nest_Lvl1 = new CrazyNestedL1();
nest_Lvl1.nest_Lvl2 = new CrazyNestedL2();
@@ -367,34 +383,41 @@ public class CollectionDataSets {
nest_Lvl1.nest_Lvl2.nest_Lvl3.nest_Lvl4.f1nal = set;
}
}
+
public static class CrazyNestedL1 {
public String a;
public int b;
public CrazyNestedL2 nest_Lvl2;
}
+
public static class CrazyNestedL2 {
public CrazyNestedL3 nest_Lvl3;
}
+
public static class CrazyNestedL3 {
public CrazyNestedL4 nest_Lvl4;
}
+
public static class CrazyNestedL4 {
public String f1nal;
}
-
+
// Copied from TypeExtractorTest
public static class FromTuple extends Tuple3<String, String, Long> {
private static final long serialVersionUID = 1L;
public int special;
}
-
+
public static class FromTupleWithCTor extends FromTuple {
- public FromTupleWithCTor() {}
- public FromTupleWithCTor(int special, long tupleField ) {
+ public FromTupleWithCTor() {
+ }
+
+ public FromTupleWithCTor(int special, long tupleField) {
this.special = special;
this.setField(tupleField, 2);
}
}
+
public static DataSet<FromTupleWithCTor> getPojoExtendingFromTuple(ExecutionEnvironment env) {
List<FromTupleWithCTor> data = new ArrayList<FromTupleWithCTor>();
data.add(new FromTupleWithCTor(1, 10L)); // 3x
@@ -404,20 +427,23 @@ public class CollectionDataSets {
data.add(new FromTupleWithCTor(2, 20L));
return env.fromCollection(data);
}
-
+
public static class PojoContainingTupleAndWritable {
public int someInt;
public String someString;
public IntWritable hadoopFan;
public Tuple2<Long, Long> theTuple;
- public PojoContainingTupleAndWritable() {}
+
+ public PojoContainingTupleAndWritable() {
+ }
+
public PojoContainingTupleAndWritable(int i, long l1, long l2) {
hadoopFan = new IntWritable(i);
someInt = i;
theTuple = new Tuple2<Long, Long>(l1, l2);
}
}
-
+
public static DataSet<PojoContainingTupleAndWritable> getPojoContainingTupleAndWritable(ExecutionEnvironment env) {
List<PojoContainingTupleAndWritable> data = new ArrayList<PojoContainingTupleAndWritable>();
data.add(new PojoContainingTupleAndWritable(1, 10L, 100L)); // 1x
@@ -428,14 +454,27 @@ public class CollectionDataSets {
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
return env.fromCollection(data);
}
-
- public static DataSet<Tuple3<Integer,CrazyNested, POJO>> getTupleContainingPojos(ExecutionEnvironment env) {
- List<Tuple3<Integer,CrazyNested, POJO>> data = new ArrayList<Tuple3<Integer,CrazyNested, POJO>>();
- data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) )); // 3x
- data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) ));
- data.add(new Tuple3<Integer,CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) ));
+
+
+
+ public static DataSet<PojoContainingTupleAndWritable> getGroupSortedPojoContainingTupleAndWritable(ExecutionEnvironment env) {
+ List<PojoContainingTupleAndWritable> data = new ArrayList<PojoContainingTupleAndWritable>();
+ data.add(new PojoContainingTupleAndWritable(1, 10L, 100L)); // 1x
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 200L)); // 5x
+ data.add(new PojoContainingTupleAndWritable(2, 20L, 201L));
+ data.add(new PojoContainingTupleAndWritable(2, 30L, 200L));
+ data.add(new PojoContainingTupleAndWritable(2, 30L, 600L));
+ data.add(new PojoContainingTupleAndWritable(2, 30L, 400L));
+ return env.fromCollection(data);
+ }
+
+ public static DataSet<Tuple3<Integer, CrazyNested, POJO>> getTupleContainingPojos(ExecutionEnvironment env) {
+ List<Tuple3<Integer, CrazyNested, POJO>> data = new ArrayList<Tuple3<Integer, CrazyNested, POJO>>();
+ data.add(new Tuple3<Integer, CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L))); // 3x
+ data.add(new Tuple3<Integer, CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
+ data.add(new Tuple3<Integer, CrazyNested, POJO>(1, new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
// POJO is not initialized according to the first two fields.
- data.add(new Tuple3<Integer,CrazyNested, POJO>(2, new CrazyNested("two", "duo", 2L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) )); // 1x
+ data.add(new Tuple3<Integer, CrazyNested, POJO>(2, new CrazyNested("two", "duo", 2L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L))); // 1x
return env.fromCollection(data);
}
@@ -443,32 +482,41 @@ public class CollectionDataSets {
public String a;
public String b;
}
+
public static class Pojo2 {
public String a2;
public String b2;
}
+
public static class PojoWithMultiplePojos {
public Pojo1 p1;
public Pojo2 p2;
public Integer i0;
- public PojoWithMultiplePojos() {}
+
+ public PojoWithMultiplePojos() {
+ }
+
public PojoWithMultiplePojos(String a, String b, String a1, String b1, Integer i0) {
p1 = new Pojo1();
p1.a = a;
p1.b = b;
p2 = new Pojo2();
p2.a2 = a1;
- p2.a2 = b1;
+ p2.b2 = b1;
this.i0 = i0;
}
}
-
+
public static DataSet<PojoWithMultiplePojos> getPojoWithMultiplePojos(ExecutionEnvironment env) {
List<PojoWithMultiplePojos> data = new ArrayList<PojoWithMultiplePojos>();
- data.add(new PojoWithMultiplePojos("a","aa","b","bb", 1));
- data.add(new PojoWithMultiplePojos("b","bb","c","cc", 2));
- data.add(new PojoWithMultiplePojos("d","dd","e","ee", 3));
+ data.add(new PojoWithMultiplePojos("a", "aa", "b", "bb", 1));
+ data.add(new PojoWithMultiplePojos("b", "bb", "c", "cc", 2));
+ data.add(new PojoWithMultiplePojos("b", "bb", "c", "cc", 2));
+ data.add(new PojoWithMultiplePojos("b", "bb", "c", "cc", 2));
+ data.add(new PojoWithMultiplePojos("d", "dd", "e", "ee", 3));
+ data.add(new PojoWithMultiplePojos("d", "dd", "e", "ee", 3));
return env.fromCollection(data);
}
-
+
}
+
[03/11] git commit: Various fixes - improved error messages - the
composite vs atomic bug aljoscha found - Comparator test for Pojo Comparator
enabled - TODO removed - string-based key expression for group sorting fields
definition - support for specifyi
Posted by rm...@apache.org.
Various fixes
- improved error messages
- the composite vs atomic bug aljoscha found
- Comparator test for Pojo Comparator enabled
- TODO removed
- string-based key expression for group sorting fields definition
- support for specifying "select all" using * and now also _ (for scala fans)
- Exception if user is having multiple fields with the same name in the class
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/fd0be2ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/fd0be2ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/fd0be2ff
Branch: refs/heads/master
Commit: fd0be2ff13494ee3739f77b1a4368b5644c55cb8
Parents: 6b493fb
Author: Robert Metzger <rm...@apache.org>
Authored: Mon Oct 6 17:11:37 2014 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Wed Oct 8 11:39:01 2014 +0200
----------------------------------------------------------------------
.../api/common/typeutils/CompositeType.java | 2 +-
.../examples/java/wordcount/PojoExample.java | 2 +-
.../api/java/operators/GroupReduceOperator.java | 16 +-
.../apache/flink/api/java/operators/Keys.java | 7 +-
.../api/java/operators/SortedGrouping.java | 75 ++++++-
.../api/java/operators/UnsortedGrouping.java | 17 +-
.../flink/api/java/typeutils/PojoTypeInfo.java | 10 +-
.../api/java/typeutils/TupleTypeInfoBase.java | 7 +-
.../flink/api/java/typeutils/TypeExtractor.java | 56 ++++--
.../flink/api/java/operators/KeysTest.java | 74 ++++++-
.../type/extractor/PojoTypeExtractionTest.java | 12 ++
.../java/type/extractor/TypeExtractorTest.java | 1 -
.../typeutils/runtime/PojoComparatorTest.java | 6 +-
.../EnumTriangleOptITCase.java | 2 +-
.../exampleScalaPrograms/PageRankITCase.java | 2 +-
.../javaApiOperators/GroupReduceITCase.java | 193 ++++++++++++++++++-
.../util/CollectionDataSets.java | 52 +++++
17 files changed, 466 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
index 1522ed1..51ad548 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/typeutils/CompositeType.java
@@ -85,7 +85,7 @@ public abstract class CompositeType<T> extends TypeInformation<T> {
&& logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1) ) // check if logical field + lookahead could contain our key
) {
// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
- addCompareField(localFieldId, ((CompositeType<?>) localFieldType).createComparator(new int[] {logicalKeyField}, orders, logicalField));
+ addCompareField(localFieldId, ((CompositeType<?>) localFieldType).createComparator(new int[] {logicalKeyField}, new boolean[] {orders[logicalKeyFieldIndex]}, logicalField));
}
// maintain logicalField
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
index a79462e..363c7a3 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/PojoExample.java
@@ -35,7 +35,7 @@ import org.apache.flink.util.Collector;
public class PojoExample {
/**
- * This is the POJO (Plain Old Java Object) that is bein used
+ * This is the POJO (Plain Old Java Object) that is being used
* for all the operations.
* As long as all fields are public or have a getter/setter, the system can handle them
*/
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
index cbcc367..1cd85c5 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/GroupReduceOperator.java
@@ -139,8 +139,7 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return po;
}
- else if (grouper.getKeys() instanceof Keys.ExpressionKeys) { //was field position key
- //TODO ask stephan
+ else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
@@ -167,19 +166,6 @@ public class GroupReduceOperator<IN, OUT> extends SingleInputUdfOperator<IN, OUT
return po;
}
-// else if (grouper.getKeys() instanceof Keys.ExpressionKeys) {
-//
-// int[] logicalKeyPositions = grouper.getKeys().computeLogicalKeyPositions();
-// UnaryOperatorInformation<IN, OUT> operatorInfo = new UnaryOperatorInformation<IN, OUT>(getInputType(), getResultType());
-// GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>> po =
-// new GroupReduceOperatorBase<IN, OUT, GroupReduceFunction<IN, OUT>>(function, operatorInfo, logicalKeyPositions, name);
-//
-// po.setCombinable(combinable);
-// po.setInput(input);
-// po.setDegreeOfParallelism(this.getParallelism());
-//
-// return po;
-// }
else {
throw new UnsupportedOperationException("Unrecognized key type.");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
index 653a04a..482370e 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/Keys.java
@@ -160,6 +160,7 @@ public abstract class Keys<T> {
public static class ExpressionKeys<T> extends Keys<T> {
public static final String SELECT_ALL_CHAR = "*";
+ public static final String SELECT_ALL_CHAR_SCALA = "_";
/**
* Flattened fields representing keys fields
@@ -245,7 +246,9 @@ public abstract class Keys<T> {
*/
public ExpressionKeys(String[] expressionsIn, TypeInformation<T> type) {
if(!(type instanceof CompositeType<?>)) {
- throw new IllegalArgumentException("Type "+type+" is not a composite type. Key expressions are not supported.");
+ throw new IllegalArgumentException("Type "+type+" is not a composite type. "
+ + "Key expressions are only supported on POJO types and Tuples. "
+ + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
}
CompositeType<T> cType = (CompositeType<T>) type;
@@ -259,7 +262,7 @@ public abstract class Keys<T> {
List<FlatFieldDescriptor> keys = new ArrayList<FlatFieldDescriptor>(); // use separate list to do a size check
cType.getKey(expressions[i], 0, keys);
if(keys.size() == 0) {
- throw new IllegalArgumentException("Unable to extract key from expression "+expressions[i]+" on key "+cType);
+ throw new IllegalArgumentException("Unable to extract key from expression '"+expressions[i]+"' on key "+cType);
}
keyFields.addAll(keys);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
index 78fd48b..c9700ce 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortedGrouping.java
@@ -19,15 +19,20 @@
package org.apache.flink.api.java.operators;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.java.functions.FirstReducer;
+
import java.util.Arrays;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+import com.google.common.base.Preconditions;
+
/**
* SortedGrouping is an intermediate step for a transformation on a grouped and sorted DataSet.<br/>
@@ -43,6 +48,9 @@ public class SortedGrouping<T> extends Grouping<T> {
private int[] groupSortKeyPositions;
private Order[] groupSortOrders ;
+ /*
+ * int sorting keys for tuples
+ */
public SortedGrouping(DataSet<T> set, Keys<T> keys, int field, Order order) {
super(set, keys);
@@ -52,10 +60,27 @@ public class SortedGrouping<T> extends Grouping<T> {
if (field >= dataSet.getType().getArity()) {
throw new IllegalArgumentException("Order key out of tuple bounds.");
}
+ // use int-based expression key to properly resolve nested tuples for grouping
+ ExpressionKeys<T> ek = new ExpressionKeys<T>(new int[]{field}, dataSet.getType());
+ this.groupSortKeyPositions = ek.computeLogicalKeyPositions();
+ this.groupSortOrders = new Order[groupSortKeyPositions.length];
+ Arrays.fill(this.groupSortOrders, order);
+ }
+
+ /*
+ * String sorting for Pojos and tuples
+ */
+ public SortedGrouping(DataSet<T> set, Keys<T> keys, String field, Order order) {
+ super(set, keys);
- this.groupSortKeyPositions = new int[]{field};
- this.groupSortOrders = new Order[]{order};
-
+ if (!(dataSet.getType() instanceof CompositeType)) {
+ throw new InvalidProgramException("Specifying order keys via field positions is only valid for composite data types (pojo / tuple / case class)");
+ }
+ // resolve String-field to int using the expression keys
+ ExpressionKeys<T> ek = new ExpressionKeys<T>(new String[]{field}, dataSet.getType());
+ this.groupSortKeyPositions = ek.computeLogicalKeyPositions();
+ this.groupSortOrders = new Order[groupSortKeyPositions.length];
+ Arrays.fill(this.groupSortOrders, order); // if field == "*"
}
protected int[] getGroupSortKeyPositions() {
@@ -108,7 +133,7 @@ public class SortedGrouping<T> extends Grouping<T> {
/**
* Sorts {@link org.apache.flink.api.java.tuple.Tuple} elements within a group on the specified field in the specified {@link Order}.</br>
- * <b>Note: Only groups of Tuple elements can be sorted.</b><br/>
+ * <b>Note: Only groups of Tuple or Pojo elements can be sorted.</b><br/>
* Groups can be sorted by multiple fields by chaining {@link #sortGroup(int, Order)} calls.
*
* @param field The Tuple field on which the group is sorted.
@@ -120,22 +145,52 @@ public class SortedGrouping<T> extends Grouping<T> {
*/
public SortedGrouping<T> sortGroup(int field, Order order) {
- int pos;
-
if (!dataSet.getType().isTupleType()) {
throw new InvalidProgramException("Specifying order keys via field positions is only valid for tuple data types");
}
if (field >= dataSet.getType().getArity()) {
throw new IllegalArgumentException("Order key out of tuple bounds.");
}
+ ExpressionKeys<T> ek = new ExpressionKeys<T>(new int[]{field}, dataSet.getType());
+ addSortGroupInternal(ek, order);
+ return this;
+ }
+
+ private void addSortGroupInternal(ExpressionKeys<T> ek, Order order) {
+ Preconditions.checkArgument(order != null, "Order can not be null");
+ int[] additionalKeyPositions = ek.computeLogicalKeyPositions();
- int newLength = this.groupSortKeyPositions.length + 1;
+ int newLength = this.groupSortKeyPositions.length + additionalKeyPositions.length;
this.groupSortKeyPositions = Arrays.copyOf(this.groupSortKeyPositions, newLength);
this.groupSortOrders = Arrays.copyOf(this.groupSortOrders, newLength);
- pos = newLength - 1;
+ int pos = newLength - additionalKeyPositions.length;
+ int off = newLength - additionalKeyPositions.length;
+ for(;pos < newLength; pos++) {
+ this.groupSortKeyPositions[pos] = additionalKeyPositions[pos - off];
+ this.groupSortOrders[pos] = order; // use the same order
+ }
+ }
+
+ /**
+ * Sorts {@link org.apache.flink.api.java.tuple.Tuple} or POJO elements within a group on the specified field in the specified {@link Order}.</br>
+ * <b>Note: Only groups of Tuple or Pojo elements can be sorted.</b><br/>
+ * Groups can be sorted by multiple fields by chaining {@link #sortGroup(String, Order)} calls.
+ *
+ * @param field The Tuple or Pojo field on which the group is sorted.
+ * @param order The Order in which the specified field is sorted.
+ * @return A SortedGrouping with specified order of group element.
+ *
+ * @see org.apache.flink.api.java.tuple.Tuple
+ * @see Order
+ */
+ public SortedGrouping<T> sortGroup(String field, Order order) {
- this.groupSortKeyPositions[pos] = field;
- this.groupSortOrders[pos] = order;
+ if (! (dataSet.getType() instanceof CompositeType)) {
+ throw new InvalidProgramException("Specifying order keys via field positions is only valid for composite data types (pojo / tuple / case class)");
+ }
+ ExpressionKeys<T> ek = new ExpressionKeys<T>(new String[]{field}, dataSet.getType());
+ addSortGroupInternal(ek, order);
return this;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
index 71b1828..910846d 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/UnsortedGrouping.java
@@ -196,7 +196,7 @@ public class UnsortedGrouping<T> extends Grouping<T> {
/**
* Sorts {@link org.apache.flink.api.java.tuple.Tuple} elements within a group on the specified field in the specified {@link Order}.</br>
- * <b>Note: Only groups of Tuple elements can be sorted.</b><br/>
+ * <b>Note: Only groups of Tuple elements and Pojos can be sorted.</b><br/>
* Groups can be sorted by multiple fields by chaining {@link #sortGroup(int, Order)} calls.
*
* @param field The Tuple field on which the group is sorted.
@@ -210,4 +210,19 @@ public class UnsortedGrouping<T> extends Grouping<T> {
return new SortedGrouping<T>(this.dataSet, this.keys, field, order);
}
+ /**
+ * Sorts Pojos within a group on the specified field in the specified {@link Order}.</br>
+ * <b>Note: Only groups of Tuple elements and Pojos can be sorted.</b><br/>
+ * Groups can be sorted by multiple fields by chaining {@link #sortGroup(String, Order)} calls.
+ *
+ * @param field The Tuple or Pojo field on which the group is sorted.
+ * @param order The Order in which the specified field is sorted.
+ * @return A SortedGrouping with specified order of group element.
+ *
+ * @see Order
+ */
+ public SortedGrouping<T> sortGroup(String field, Order order) {
+ return new SortedGrouping<T>(this.dataSet, this.keys, field, order);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
index fba1f24..7d6e4ec 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/PojoTypeInfo.java
@@ -118,7 +118,7 @@ public class PojoTypeInfo<T> extends CompositeType<T>{
@Override
public void getKey(String fieldExpression, int offset, List<FlatFieldDescriptor> result) {
// handle 'select all' first
- if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR)) {
+ if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR) || fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR_SCALA)) {
int keyPosition = 0;
for(PojoField field : fields) {
if(field.type instanceof AtomicType) {
@@ -145,6 +145,10 @@ public class PojoTypeInfo<T> extends CompositeType<T>{
fieldId += fields[i].type.getTotalFields()-1;
}
if (fields[i].field.getName().equals(fieldExpression)) {
+ if(fields[i].type instanceof CompositeType) {
+ throw new IllegalArgumentException("The specified field '"+fieldExpression+"' is refering to a composite type.\n"
+ + "Either select all elements in this type with the '"+ExpressionKeys.SELECT_ALL_CHAR+"' operator or specify a field in the sub-type");
+ }
result.add(new FlatFieldDescriptor(offset + fieldId, fields[i].type));
return;
}
@@ -158,13 +162,13 @@ public class PojoTypeInfo<T> extends CompositeType<T>{
for (int i = 0; i < fields.length; i++) {
if (fields[i].field.getName().equals(firstField)) {
if (!(fields[i].type instanceof CompositeType<?>)) {
- throw new RuntimeException("Field "+fields[i].type+" is not composite type");
+ throw new RuntimeException("Field "+fields[i].type+" (specified by '"+fieldExpression+"') is not a composite type");
}
CompositeType<?> cType = (CompositeType<?>) fields[i].type;
cType.getKey(rest, offset + fieldId, result); // recurse
return;
}
- fieldId++;
+ fieldId += fields[i].type.getTotalFields();
}
throw new RuntimeException("Unable to find field "+fieldExpression+" in type "+this+" (looking for '"+firstField+"')");
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
index 4babbd7..dc75b2c 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TupleTypeInfoBase.java
@@ -95,7 +95,7 @@ public abstract class TupleTypeInfoBase<T> extends CompositeType<T> {
@Override
public void getKey(String fieldExpression, int offset, List<FlatFieldDescriptor> result) {
// handle 'select all'
- if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR)) {
+ if(fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR) || fieldExpression.equals(ExpressionKeys.SELECT_ALL_CHAR_SCALA)) {
int keyPosition = 0;
for(TypeInformation<?> type : types) {
if(type instanceof AtomicType) {
@@ -157,7 +157,10 @@ public abstract class TupleTypeInfoBase<T> extends CompositeType<T> {
for(int i = 0; i < pos; i++) {
offset += types[i].getTotalFields() - 1; // this adds only something to offset if its a composite type.
}
-
+ if(types[pos] instanceof CompositeType) {
+ throw new IllegalArgumentException("The specified field '"+fieldExpression+"' is refering to a composite type.\n"
+ + "Either select all elements in this type with the '"+ExpressionKeys.SELECT_ALL_CHAR+"' operator or specify a field in the sub-type");
+ }
result.add(new FlatFieldDescriptor(offset + pos, types[pos]));
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 5d216e9..5935e31 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -337,7 +337,7 @@ public class TypeExtractor {
int fieldCount = countFieldsInClass(tAsClass);
if(fieldCount != tupleSubTypes.length) {
// the class is not a real tuple because it contains additional fields. treat as a pojo
- return (TypeInformation<OUT>) analyzePojo(tAsClass, new ArrayList<Type>() ); // the typeHierarchy here should be sufficient, even though it stops at the Tuple.class.
+ return (TypeInformation<OUT>) analyzePojo(tAsClass, new ArrayList<Type>(), null); // the typeHierarchy here should be sufficient, even though it stops at the Tuple.class.
}
return new TupleTypeInfo(tAsClass, tupleSubTypes);
@@ -399,7 +399,7 @@ public class TypeExtractor {
}
// objects with generics are treated as raw type
else if (t instanceof ParameterizedType) { //TODO
- return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), typeHierarchy);
+ return privateGetForClass((Class<OUT>) ((ParameterizedType) t).getRawType(), typeHierarchy, (ParameterizedType) t);
}
// no tuple, no TypeVariable, no generic type
else if (t instanceof Class) {
@@ -859,8 +859,11 @@ public class TypeExtractor {
return new TypeExtractor().privateGetForClass(clazz, new ArrayList<Type>());
}
- @SuppressWarnings("unchecked")
private <X> TypeInformation<X> privateGetForClass(Class<X> clazz, ArrayList<Type> typeHierarchy) {
+ return privateGetForClass(clazz, typeHierarchy, null);
+ }
+ @SuppressWarnings("unchecked")
+ private <X> TypeInformation<X> privateGetForClass(Class<X> clazz, ArrayList<Type> typeHierarchy, ParameterizedType clazzTypeHint) {
Validate.notNull(clazz);
// check for abstract classes or interfaces
@@ -931,7 +934,7 @@ public class TypeExtractor {
// this is a type generated by Avro. GenericTypeInfo is able to handle this case because its using Avro.
return new GenericTypeInfo<X>(clazz);
}
- TypeInformation<X> pojoType = analyzePojo(clazz, typeHierarchy);
+ TypeInformation<X> pojoType = analyzePojo(clazz, typeHierarchy, clazzTypeHint);
if (pojoType != null) {
return pojoType;
}
@@ -968,7 +971,7 @@ public class TypeExtractor {
// check for getter
if( // The name should be "get<FieldName>" or "<fieldName>" (for scala).
- (m.getName().toLowerCase().contains("get"+fieldNameLow) || m.getName().toLowerCase().contains(fieldNameLow)) &&
+ (m.getName().toLowerCase().equals("get"+fieldNameLow) || m.getName().toLowerCase().equals(fieldNameLow)) &&
// no arguments for the getter
m.getParameterTypes().length == 0 &&
// return type is same as field type (or the generic variant of it)
@@ -980,7 +983,7 @@ public class TypeExtractor {
hasGetter = true;
}
// check for setters (<FieldName>_$eq for scala)
- if((m.getName().toLowerCase().contains("set"+fieldNameLow) || m.getName().toLowerCase().contains(fieldNameLow+"_$eq")) &&
+ if((m.getName().toLowerCase().equals("set"+fieldNameLow) || m.getName().toLowerCase().equals(fieldNameLow+"_$eq")) &&
m.getParameterTypes().length == 1 && // one parameter of the field's type
( m.getParameterTypes()[0].equals( fieldType ) || (fieldTypeGeneric != null && m.getGenericParameterTypes()[0].equals(fieldTypeGeneric) ) )&&
// return type is void.
@@ -1006,13 +1009,16 @@ public class TypeExtractor {
}
}
- private <X> TypeInformation<X> analyzePojo(Class<X> clazz, ArrayList<Type> typeHierarchy) {
+ private <X> TypeInformation<X> analyzePojo(Class<X> clazz, ArrayList<Type> typeHierarchy, ParameterizedType clazzTypeHint) {
// try to create Type hierarchy, if the incoming one is empty.
if(typeHierarchy.size() == 0) {
recursivelyGetTypeHierarchy(typeHierarchy, clazz, Object.class);
}
+ if(clazzTypeHint != null) {
+ recursivelyGetTypeHierarchy(typeHierarchy, clazzTypeHint, Object.class);
+ }
- List<Field> fields = removeNonObjectFields(getAllDeclaredFields(clazz));
+ List<Field> fields = getAllDeclaredFields(clazz);
List<PojoField> pojoFields = new ArrayList<PojoField>();
for (Field field : fields) {
Type fieldType = field.getGenericType();
@@ -1057,32 +1063,40 @@ public class TypeExtractor {
return pojoType;
}
- // recursively determine all declared fields
- private static List<Field> getAllDeclaredFields(Class<?> clazz) {
+ /**
+ * recursively determine all declared fields
+ * This is required because getFields() is not returning
+ */
+ public static List<Field> getAllDeclaredFields(Class<?> clazz) {
List<Field> result = new ArrayList<Field>();
while (clazz != null) {
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
+ if(Modifier.isTransient(field.getModifiers()) || Modifier.isStatic(field.getModifiers())) {
+ continue; // we have no use for transient or static fields
+ }
+ if(hasFieldWithSameName(field.getName(), result)) {
+ throw new RuntimeException("The field "+field+" is already contained in the hierarchy of the class "+clazz+"."
+ + "Please use unique field names through your classes hierarchy");
+ }
result.add(field);
}
clazz = clazz.getSuperclass();
}
return result;
}
-
- /**
- * Remove transient and static fields from a list of fields.
- */
- private static List<Field> removeNonObjectFields(List<Field> fields) {
- List<Field> result = new ArrayList<Field>();
- for(Field field: fields) {
- if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) {
- result.add(field);
+
+ private static boolean hasFieldWithSameName(String name, List<Field> fields) {
+ for(Field field : fields) {
+ if(name.equals(field.getName())) {
+ return true;
}
}
- return result;
+ return false;
}
+
+
// recursively determine all declared methods
private static List<Method> getAllDeclaredMethods(Class<?> clazz) {
List<Method> result = new ArrayList<Method>();
@@ -1111,7 +1125,7 @@ public class TypeExtractor {
int numFields = t.getArity();
if(numFields != countFieldsInClass(value.getClass())) {
// not a tuple since it has more fields.
- return analyzePojo((Class<X>) value.getClass(), new ArrayList<Type>()); // we immediately call analyze Pojo here, because
+ return analyzePojo((Class<X>) value.getClass(), new ArrayList<Type>(), null); // we immediately call analyze Pojo here, because
// there is currently no other type that can handle such a class.
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
index 00ab520..37a117f 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operators/KeysTest.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.flink.api.java.operators;
import java.lang.reflect.InvocationTargetException;
@@ -24,10 +23,13 @@ import java.util.Arrays;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.type.extractor.PojoTypeExtractionTest.ComplexNestedClass;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -93,7 +95,7 @@ public class KeysTest {
}
@Test
- public void testInvalid() throws Throwable {
+ public void testInvalidTuple() throws Throwable {
TupleTypeInfo<Tuple3<String, Tuple3<String, String, String>, String>> typeInfo = new TupleTypeInfo<Tuple3<String,Tuple3<String,String,String>,String>>(
BasicTypeInfo.STRING_TYPE_INFO,
new TupleTypeInfo<Tuple3<String, String, String>>(BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO),
@@ -101,7 +103,12 @@ public class KeysTest {
ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>> fpk;
String[][] tests = new String[][] {
- new String[] {"f11"},new String[] {"f-35"}, new String[] {"f0.f33"}, new String[] {"f1.f33"}
+ new String[] {"f0.f1"}, // nesting into unnested
+ new String[] {"f11"},
+ new String[] {"f-35"},
+ new String[] {"f0.f33"},
+ new String[] {"f1.f33"},
+ new String[] {"f1"} // select full tuple without saying "f1.*"
};
for(int i = 0; i < tests.length; i++) {
Throwable e = null;
@@ -115,6 +122,28 @@ public class KeysTest {
}
}
+ @Test
+ public void testInvalidPojo() throws Throwable {
+ TypeInformation<ComplexNestedClass> ti = TypeExtractor.getForClass(ComplexNestedClass.class);
+ ExpressionKeys<ComplexNestedClass> ek;
+
+ String[][] tests = new String[][] {
+ new String[] {"nonexistent"},
+ new String[] {"date.abc"}, // nesting into unnested
+ new String[] {"word"} // select full tuple without saying "f1.*"
+ };
+ for(int i = 0; i < tests.length; i++) {
+ Throwable e = null;
+ try {
+ ek = new ExpressionKeys<ComplexNestedClass>(tests[i], ti);
+ } catch(Throwable t) {
+ // System.err.println("Message: "+t.getMessage()); t.printStackTrace();
+ e = t;
+ }
+ Assert.assertNotNull(e);
+ }
+ }
+
@Test
public void testTupleKeyExpansion() {
TupleTypeInfo<Tuple3<String, Tuple3<String, String, String>, String>> typeInfo = new TupleTypeInfo<Tuple3<String,Tuple3<String,String,String>,String>>(
@@ -144,6 +173,10 @@ public class KeysTest {
fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"*"}, typeInfo);
Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+ // scala style "select all"
+ fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"_"}, typeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions());
+
// this was a bug:
fpk = new ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>>(new String[] {"f2"}, typeInfo);
Assert.assertArrayEquals(new int[] {4}, fpk.computeLogicalKeyPositions());
@@ -177,10 +210,45 @@ public class KeysTest {
complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"*"}, complexTypeInfo);
Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions());
+ // scala style select all
+ complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"_"}, complexTypeInfo);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions());
+
complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"f1.f0.*"}, complexTypeInfo);
Assert.assertArrayEquals(new int[] {1,2,3}, complexFpk.computeLogicalKeyPositions());
complexFpk = new ExpressionKeys<Tuple3<String, Tuple3<Tuple3<String, String, String>, String, String>, String>>(new String[] {"f2"}, complexTypeInfo);
Assert.assertArrayEquals(new int[] {6}, complexFpk.computeLogicalKeyPositions());
}
+
+ public static class Pojo1 {
+ public String a;
+ public String b;
+ }
+ public static class Pojo2 {
+ public String a2;
+ public String b2;
+ }
+ public static class PojoWithMultiplePojos {
+ public Pojo1 p1;
+ public Pojo2 p2;
+ public Integer i0;
+ }
+
+ @Test
+ public void testPojoKeys() {
+ TypeInformation<PojoWithMultiplePojos> ti = TypeExtractor.getForClass(PojoWithMultiplePojos.class);
+ ExpressionKeys<PojoWithMultiplePojos> ek;
+ ek = new ExpressionKeys<PojoWithMultiplePojos>(new String[]{"*"}, ti);
+ Assert.assertArrayEquals(new int[] {0,1,2,3,4}, ek.computeLogicalKeyPositions());
+
+ ek = new ExpressionKeys<PojoWithMultiplePojos>(new String[]{"p1.*"}, ti);
+ Assert.assertArrayEquals(new int[] {1,2}, ek.computeLogicalKeyPositions());
+
+ ek = new ExpressionKeys<PojoWithMultiplePojos>(new String[]{"p2.*"}, ti);
+ Assert.assertArrayEquals(new int[] {3,4}, ek.computeLogicalKeyPositions());
+
+ ek = new ExpressionKeys<PojoWithMultiplePojos>(new String[]{"i0"}, ti);
+ Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
index 34dbd99..01d32c1 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/PojoTypeExtractionTest.java
@@ -50,6 +50,14 @@ import com.google.common.collect.HashMultiset;
*/
public class PojoTypeExtractionTest {
+ public static class HasDuplicateField extends WC {
+ private int count; // duplicate
+ }
+
+ @Test(expected=RuntimeException.class)
+ public void testDuplicateFieldException() {
+ TypeExtractor.createTypeInfo(HasDuplicateField.class);
+ }
// test with correct pojo types
public static class WC { // is a pojo
@@ -210,6 +218,10 @@ public class PojoTypeExtractionTest {
}
ffd.clear();
+ // scala style full tuple selection for pojos
+ pojoType.getKey("complex.word._", 0, ffd);
+ Assert.assertEquals(3, ffd.size());
+ ffd.clear();
pojoType.getKey("complex.*", 0, ffd);
Assert.assertEquals(8, ffd.size());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
index b4b1c19..695a42e 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/type/extractor/TypeExtractorTest.java
@@ -1247,7 +1247,6 @@ public class TypeExtractorTest {
public static class InType extends MyObject<String> {}
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
-// @Ignore
public void testParamertizedCustomObject() {
RichMapFunction<?, ?> function = new RichMapFunction<InType, MyObject<String>>() {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
index 61f6167..e53f48a 100644
--- a/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/java/typeutils/runtime/PojoComparatorTest.java
@@ -28,10 +28,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.operators.Keys.ExpressionKeys;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.junit.Assert;
-import org.junit.Ignore;
-//@Ignore // TODO
public class PojoComparatorTest extends ComparatorTestBase<PojoContainingTuple> {
TypeInformation<PojoContainingTuple> type = TypeExtractor.getForClass(PojoContainingTuple.class);
@@ -39,7 +37,7 @@ public class PojoComparatorTest extends ComparatorTestBase<PojoContainingTuple>
new PojoContainingTuple(1, 1L, 1L),
new PojoContainingTuple(2, 2L, 2L),
new PojoContainingTuple(8519, 85190L, 85190L),
- new PojoContainingTuple(-51498, 85191L, 85191L),
+ new PojoContainingTuple(8520, 85191L, 85191L),
};
@Override
@@ -48,7 +46,7 @@ public class PojoComparatorTest extends ComparatorTestBase<PojoContainingTuple>
CompositeType<PojoContainingTuple> cType = (CompositeType<PojoContainingTuple>) type;
ExpressionKeys<PojoContainingTuple> keys = new ExpressionKeys<PojoContainingTuple>(new String[] {"theTuple.*"}, cType);
boolean[] orders = new boolean[keys.getNumberOfKeyFields()];
- Arrays.fill(orders, true);
+ Arrays.fill(orders, ascending);
return cType.createComparator(keys.computeLogicalKeyPositions(), orders, 0);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
index 4f00b1d..9701086 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/EnumTriangleOptITCase.java
@@ -18,7 +18,7 @@
package org.apache.flink.test.exampleScalaPrograms;
-import org.apache.flink.examples.java.graph.EnumTrianglesOpt;
+import org.apache.flink.examples.scala.graph.EnumTrianglesOpt;
import org.apache.flink.test.testdata.EnumTriangleData;
import org.apache.flink.test.util.JavaProgramTestBase;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
index 42c6a8e..bc026c9 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/exampleScalaPrograms/PageRankITCase.java
@@ -24,7 +24,7 @@ import java.util.Collection;
import java.util.LinkedList;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.examples.java.graph.PageRankBasic;
+import org.apache.flink.examples.scala.graph.PageRankBasic;
import org.apache.flink.test.testdata.PageRankData;
import org.apache.flink.test.util.JavaProgramTestBase;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
index fce56d1..63c4dfc 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/GroupReduceITCase.java
@@ -53,7 +53,7 @@ import org.apache.flink.api.java.ExecutionEnvironment;
@RunWith(Parameterized.class)
public class GroupReduceITCase extends JavaProgramTestBase {
- private static int NUM_PROGRAMS = 19;
+ private static int NUM_PROGRAMS = 26;
private int curProgId = config.getInteger("ProgramId", -1);
private String resultPath;
@@ -598,7 +598,175 @@ public class GroupReduceITCase extends JavaProgramTestBase {
// return expected result
return "3\n1\n";
- }
+ }
+ case 20: {
+ /*
+ * Test string-based definition on group sort, based on test:
+ * check correctness of groupReduce with descending group sort
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> reduceDs = ds.
+ groupBy(1).sortGroup("f2", Order.DESCENDING).reduceGroup(new Tuple3SortedGroupReduce());
+
+ reduceDs.writeAsCsv(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1,1,Hi\n" +
+ "5,2,Hello world-Hello\n" +
+ "15,3,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
+ "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" +
+ "65,5,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
+ "111,6,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";
+
+ }
+ case 21: {
+ /*
+ * Test int-based definition on group sort, for (full) nested Tuple
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
+ DataSet<String> reduceDs = ds.groupBy("f1").sortGroup(0, Order.DESCENDING).reduceGroup(new NestedTupleReducer());
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "a--(1,1)-(1,2)-(1,3)-\n" +
+ "b--(2,2)-\n"+
+ "c--(3,3)-(3,6)-(3,9)-\n";
+ }
+ case 22: {
+ /*
+ * Test int-based definition on group sort, for (partial) nested Tuple ASC
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
+ // f0.f0 is first integer
+ DataSet<String> reduceDs = ds.groupBy("f1").sortGroup("f0.f0", Order.ASCENDING).reduceGroup(new NestedTupleReducer());
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "a--(1,3)-(1,2)-(2,1)-\n" +
+ "b--(2,2)-\n"+
+ "c--(3,3)-(3,6)-(4,9)-\n";
+ }
+ case 23: {
+ /*
+ * Test string-based definition on group sort, for (partial) nested Tuple DESC
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
+ // f0.f0 is first integer
+ DataSet<String> reduceDs = ds.groupBy("f1").sortGroup("f0.f0", Order.DESCENDING).reduceGroup(new NestedTupleReducer());
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "a--(2,1)-(1,3)-(1,2)-\n" +
+ "b--(2,2)-\n"+
+ "c--(4,9)-(3,3)-(3,6)-\n";
+ }
+ case 24: {
+ /*
+ * Test string-based definition on group sort, for two grouping keys
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
+ // f0.f0 is first integer
+ DataSet<String> reduceDs = ds.groupBy("f1").sortGroup("f0.f0", Order.DESCENDING).sortGroup("f0.f1", Order.DESCENDING).reduceGroup(new NestedTupleReducer());
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "a--(2,1)-(1,3)-(1,2)-\n" +
+ "b--(2,2)-\n"+
+ "c--(4,9)-(3,6)-(3,3)-\n";
+ }
+ case 25: {
+ /*
+ * Test string-based definition on group sort, for two grouping keys with Pojos
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getGroupSortedPojoContainingTupleAndWritable(env);
+ // f0.f0 is first integer
+ DataSet<String> reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING)
+ .reduceGroup(new GroupReduceFunction<CollectionDataSets.PojoContainingTupleAndWritable, String>() {
+ @Override
+ public void reduce(
+ Iterable<PojoContainingTupleAndWritable> values,
+ Collector<String> out) throws Exception {
+ boolean once = false;
+ StringBuilder concat = new StringBuilder();
+ for(PojoContainingTupleAndWritable value : values) {
+ if(!once) {
+ concat.append(value.hadoopFan.get());
+ concat.append("---");
+ once = true;
+ }
+ concat.append(value.theTuple);
+ concat.append("-");
+ }
+ out.collect(concat.toString());
+ }
+ });
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1---(10,100)-\n" +
+ "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n";
+ }
+ case 26: {
+ /*
+ * Test grouping with pojo containing multiple pojos (was a bug)
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setDegreeOfParallelism(1);
+
+ DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getPojoWithMultiplePojos(env);
+ // f0.f0 is first integer
+ DataSet<String> reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING)
+ .reduceGroup(new GroupReduceFunction<CollectionDataSets.PojoContainingTupleAndWritable, String>() {
+ @Override
+ public void reduce(
+ Iterable<PojoContainingTupleAndWritable> values,
+ Collector<String> out) throws Exception {
+ boolean once = false;
+ StringBuilder concat = new StringBuilder();
+ for(PojoContainingTupleAndWritable value : values) {
+ if(!once) {
+ concat.append(value.hadoopFan.get());
+ concat.append("---");
+ once = true;
+ }
+ concat.append(value.theTuple);
+ concat.append("-");
+ }
+ out.collect(concat.toString());
+ }
+ });
+ reduceDs.writeAsText(resultPath);
+ env.execute();
+
+ // return expected result
+ return "1---(10,100)-\n" +
+ "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n";
+ }
+
default: {
throw new IllegalArgumentException("Invalid program id");
}
@@ -607,6 +775,27 @@ public class GroupReduceITCase extends JavaProgramTestBase {
}
+
+ public static class NestedTupleReducer implements GroupReduceFunction<Tuple2<Tuple2<Integer,Integer>,String>, String> {
+ @Override
+ public void reduce(
+ Iterable<Tuple2<Tuple2<Integer, Integer>, String>> values,
+ Collector<String> out)
+ throws Exception {
+ boolean once = false;
+ StringBuilder concat = new StringBuilder();
+ for(Tuple2<Tuple2<Integer, Integer>, String> value : values) {
+ if(!once) {
+ concat.append(value.f1).append("--");
+ once = true;
+ }
+ concat.append(value.f0); // the tuple with the sorted groups
+ concat.append("-");
+ }
+ out.collect(concat.toString());
+ }
+ }
+
public static class Tuple3GroupReduce implements GroupReduceFunction<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fd0be2ff/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
index d023287..e5ab29a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/util/CollectionDataSets.java
@@ -155,6 +155,26 @@ public class CollectionDataSets {
return env.fromCollection(data, type);
}
+ public static DataSet<Tuple2<Tuple2<Integer, Integer>, String>> getGroupSortedNestedTupleDataSet(ExecutionEnvironment env) {
+
+ List<Tuple2<Tuple2<Integer, Integer>, String>> data = new ArrayList<Tuple2<Tuple2<Integer, Integer>, String>>();
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,3), "a"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(1,2), "a"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,1), "a"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(2,2), "b"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,3), "c"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(3,6), "c"));
+ data.add(new Tuple2<Tuple2<Integer,Integer>, String>(new Tuple2<Integer, Integer>(4,9), "c"));
+
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>> type = new
+ TupleTypeInfo<Tuple2<Tuple2<Integer, Integer>, String>>(
+ new TupleTypeInfo<Tuple2<Integer, Integer>>(BasicTypeInfo.INT_TYPE_INFO,BasicTypeInfo.INT_TYPE_INFO),
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
+ return env.fromCollection(data, type);
+ }
+
public static DataSet<String> getStringDataSet(ExecutionEnvironment env) {
List<String> data = new ArrayList<String>();
@@ -418,5 +438,37 @@ public class CollectionDataSets {
data.add(new Tuple3<Integer,CrazyNested, POJO>(2, new CrazyNested("two", "duo", 2L), new POJO(1, "First",10, 100, 1000L, "One", 10000L) )); // 1x
return env.fromCollection(data);
}
+
+ public static class Pojo1 {
+ public String a;
+ public String b;
+ }
+ public static class Pojo2 {
+ public String a2;
+ public String b2;
+ }
+ public static class PojoWithMultiplePojos {
+ public Pojo1 p1;
+ public Pojo2 p2;
+ public Integer i0;
+ public PojoWithMultiplePojos() {}
+ public PojoWithMultiplePojos(String a, String b, String a1, String b1, Integer i0) {
+ p1 = new Pojo1();
+ p1.a = a;
+ p1.b = b;
+ p2 = new Pojo2();
+ p2.a2 = a1;
+ p2.a2 = b1;
+ this.i0 = i0;
+ }
+ }
+
+ public static DataSet<PojoWithMultiplePojos> getPojoWithMultiplePojos(ExecutionEnvironment env) {
+ List<PojoWithMultiplePojos> data = new ArrayList<PojoWithMultiplePojos>();
+ data.add(new PojoWithMultiplePojos("a","aa","b","bb", 1));
+ data.add(new PojoWithMultiplePojos("b","bb","c","cc", 2));
+ data.add(new PojoWithMultiplePojos("d","dd","e","ee", 3));
+ return env.fromCollection(data);
+ }
}