You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2017/07/13 10:18:28 UTC
[19/44] flink git commit: [FLINK-6617] [table] Restructuring of tests
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
new file mode 100644
index 0000000..1561da0
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import java.math.BigDecimal
+
+import org.apache.flink.api.common.time.Time
+import org.apache.flink.api.scala._
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.{StreamQueryConfig, TableEnvironment}
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
+import org.apache.flink.table.runtime.stream.table.GroupWindowITCase._
+import org.apache.flink.table.runtime.utils.StreamITCase
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+/**
+ * We only test some aggregations until better testing of constructed DataStream
+ * programs is possible.
+ */
+class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
+ private val queryConfig = new StreamQueryConfig()
+ queryConfig.withIdleStateRetentionTime(Time.hours(1), Time.hours(2))
+
+ val data = List(
+ (1L, 1, "Hi"),
+ (2L, 2, "Hello"),
+ (4L, 2, "Hello"),
+ (8L, 3, "Hello world"),
+ (16L, 3, "Hello world"))
+
+ val data2 = List(
+ (1L, 1, 1d, 1f, new BigDecimal("1"), "Hi"),
+ (2L, 2, 2d, 2f, new BigDecimal("2"), "Hallo"),
+ (3L, 2, 2d, 2f, new BigDecimal("2"), "Hello"),
+ (4L, 5, 5d, 5f, new BigDecimal("5"), "Hello"),
+ (7L, 3, 3d, 3f, new BigDecimal("3"), "Hello"),
+ (8L, 3, 3d, 3f, new BigDecimal("3"), "Hello world"),
+ (16L, 4, 4d, 4f, new BigDecimal("4"), "Hello world"))
+
+ @Test
+ def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'proctime.proctime)
+
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+ .groupBy('w, 'string)
+ .select('string, countFun('int), 'int.avg,
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+
+ val results = windowedTable.toAppendStream[Row](queryConfig)
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq("Hello world,1,3,8,3", "Hello world,2,3,12,3", "Hello,1,2,2,2",
+ "Hello,2,2,3,2", "Hi,1,1,1,1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSessionGroupWindowOverTime(): Unit = {
+ //To verify the "merge" functionality, we create this test with the following characteristics:
+ // 1. set the Parallelism to 1, and have the test data out of order
+ // 2. create a waterMark with 10ms offset to delay the window emission by 10ms
+ val sessionWindowTestdata = List(
+ (1L, 1, "Hello"),
+ (2L, 2, "Hello"),
+ (8L, 8, "Hello"),
+ (9L, 9, "Hello World"),
+ (4L, 4, "Hello"),
+ (16L, 16, "Hello"))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvgWithMerge
+
+ val stream = env
+ .fromCollection(sessionWindowTestdata)
+ .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset[(Long, Int, String)](10L))
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'rowtime.rowtime)
+
+ val windowedTable = table
+ .window(Session withGap 5.milli on 'rowtime as 'w)
+ .groupBy('w, 'string)
+ .select('string, countFun('int), 'int.avg,
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq("Hello World,1,9,9,9", "Hello,1,16,16,16", "Hello,4,3,5,5")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'proctime.proctime)
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Tumble over 2.rows on 'proctime as 'w)
+ .groupBy('w)
+ .select(countFun('string), 'int.avg,
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+
+ val results = windowedTable.toAppendStream[Row](queryConfig)
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq("2,1,1,1", "2,2,6,2")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeTumblingWindow(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset[(Long, Int, String)](0L))
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'rowtime.rowtime)
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Tumble over 5.milli on 'rowtime as 'w)
+ .groupBy('w, 'string)
+ .select('string, countFun('string), 'int.avg, weightAvgFun('long, 'int),
+ weightAvgFun('int, 'int), 'int.min, 'int.max, 'int.sum, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hello world,1,3,8,3,3,3,3,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01",
+ "Hello world,1,3,16,3,3,3,3,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02",
+ "Hello,2,2,3,2,2,2,4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testGroupWindowWithoutKeyInProjection(): Unit = {
+ val data = List(
+ (1L, 1, "Hi", 1, 1),
+ (2L, 2, "Hello", 2, 2),
+ (4L, 2, "Hello", 2, 2),
+ (8L, 3, "Hello world", 3, 3),
+ (16L, 3, "Hello world", 3, 3))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'int2, 'int3, 'proctime.proctime)
+
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+ .groupBy('w, 'int2, 'int3, 'string)
+ .select(weightAvgFun('long, 'int))
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq("12", "8", "2", "3", "1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+
+
+ // ----------------------------------------------------------------------------------------------
+ // Sliding windows
+ // ----------------------------------------------------------------------------------------------
+
+ @Test
+ def testAllEventTimeSlidingGroupWindowOverTime(): Unit = {
+ // please keep this test in sync with the DataSet variant
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val windowedTable = table
+ .window(Slide over 5.milli every 2.milli on 'long as 'w)
+ .groupBy('w)
+ .select('int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1,1970-01-01 00:00:00.008,1970-01-01 00:00:00.013",
+ "1,1970-01-01 00:00:00.012,1970-01-01 00:00:00.017",
+ "1,1970-01-01 00:00:00.014,1970-01-01 00:00:00.019",
+ "1,1970-01-01 00:00:00.016,1970-01-01 00:00:00.021",
+ "2,1969-12-31 23:59:59.998,1970-01-01 00:00:00.003",
+ "2,1970-01-01 00:00:00.006,1970-01-01 00:00:00.011",
+ "3,1970-01-01 00:00:00.002,1970-01-01 00:00:00.007",
+ "3,1970-01-01 00:00:00.004,1970-01-01 00:00:00.009",
+ "4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSlidingGroupWindowOverTimeOverlappingFullPane(): Unit = {
+ // please keep this test in sync with the DataSet variant
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val windowedTable = table
+ .window(Slide over 10.milli every 5.milli on 'long as 'w)
+ .groupBy('w, 'string)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hallo,1,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
+ "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
+ "Hello world,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
+ "Hello world,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.015",
+ "Hello world,1,1970-01-01 00:00:00.01,1970-01-01 00:00:00.02",
+ "Hello world,1,1970-01-01 00:00:00.015,1970-01-01 00:00:00.025",
+ "Hello,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.015",
+ "Hello,2,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
+ "Hello,3,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
+ "Hi,1,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSlidingGroupWindowOverTimeOverlappingSplitPane(): Unit = {
+ // please keep this test in sync with the DataSet variant
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val windowedTable = table
+ .window(Slide over 5.milli every 4.milli on 'long as 'w)
+ .groupBy('w, 'string)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hello world,1,1970-01-01 00:00:00.004,1970-01-01 00:00:00.009",
+ "Hello world,1,1970-01-01 00:00:00.008,1970-01-01 00:00:00.013",
+ "Hello world,1,1970-01-01 00:00:00.012,1970-01-01 00:00:00.017",
+ "Hello world,1,1970-01-01 00:00:00.016,1970-01-01 00:00:00.021",
+ "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hello,2,1970-01-01 00:00:00.004,1970-01-01 00:00:00.009",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSlidingGroupWindowOverTimeNonOverlappingFullPane(): Unit = {
+ // please keep this test in sync with the DataSet variant
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val windowedTable = table
+ .window(Slide over 5.milli every 10.milli on 'long as 'w)
+ .groupBy('w, 'string)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSlidingGroupWindowOverTimeNonOverlappingSplitPane(): Unit = {
+ // please keep this test in sync with the DataSet variant
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ val table = stream.toTable(tEnv, 'long.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val windowedTable = table
+ .window(Slide over 3.milli every 10.milli on 'long as 'w)
+ .groupBy('w, 'string)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeGroupWindowWithoutExplicitTimeField(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data2)
+ .assignTimestampsAndWatermarks(
+ new TimestampAndWatermarkWithOffset[(Long, Int, Double, Float, BigDecimal, String)](0L))
+ .map(t => (t._2, t._6))
+ val table = stream.toTable(tEnv, 'int, 'string, 'rowtime.rowtime)
+
+ val windowedTable = table
+ .window(Slide over 3.milli every 10.milli on 'rowtime as 'w)
+ .groupBy('w, 'string)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+ val expected = Seq(
+ "Hallo,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.003")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
+
+object GroupWindowITCase {
+
+ class TimestampAndWatermarkWithOffset[T <: Product](
+ offset: Long) extends AssignerWithPunctuatedWatermarks[T] {
+
+ override def checkAndGetNextWatermark(
+ lastElement: T,
+ extractedTimestamp: Long): Watermark = {
+ new Watermark(extractedTimestamp - offset)
+ }
+
+ override def extractTimestamp(
+ element: T,
+ previousElementTimestamp: Long): Long = {
+ element.productElement(0).asInstanceOf[Long]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
new file mode 100644
index 0000000..73484d2
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg
+import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc0
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.functions.aggfunctions.CountAggFunction
+import org.apache.flink.table.runtime.stream.table.OverWindowITCase._
+import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class OverWindowITCase extends StreamingWithStateTestBase {
+
+ @Test
+ def testProcTimeUnBoundedPartitionedRowOver(): Unit = {
+
+ val data = List(
+ (1L, 1, "Hello"),
+ (2L, 2, "Hello"),
+ (3L, 3, "Hello"),
+ (4L, 4, "Hello"),
+ (5L, 5, "Hello"),
+ (6L, 6, "Hello"),
+ (7L, 7, "Hello World"),
+ (8L, 8, "Hello World"),
+ (20L, 20, "Hello World"))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+ StreamITCase.clear
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(
+ Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW as 'w)
+ .select('c, countFun('b) over 'w as 'mycount, weightAvgFun('a, 'b) over 'w as 'wAvg)
+ .select('c, 'mycount, 'wAvg)
+
+ val results = windowedTable.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "Hello World,1,7", "Hello World,2,7", "Hello World,3,14",
+ "Hello,1,1", "Hello,2,1", "Hello,3,2", "Hello,4,3", "Hello,5,3", "Hello,6,4")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testRowTimeUnBoundedPartitionedRangeOver(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ env.setStateBackend(getStateBackend)
+ StreamITCase.testResults = mutable.MutableList()
+ StreamITCase.clear
+ env.setParallelism(1)
+
+ val data = Seq(
+ Left(14000005L, (1, 1L, "Hi")),
+ Left(14000000L, (2, 1L, "Hello")),
+ Left(14000002L, (1, 1L, "Hello")),
+ Left(14000002L, (1, 2L, "Hello")),
+ Left(14000002L, (1, 3L, "Hello world")),
+ Left(14000003L, (2, 2L, "Hello world")),
+ Left(14000003L, (2, 3L, "Hello world")),
+ Right(14000020L),
+ Left(14000021L, (1, 4L, "Hello world")),
+ Left(14000022L, (1, 5L, "Hello world")),
+ Left(14000022L, (1, 6L, "Hello world")),
+ Left(14000022L, (1, 7L, "Hello world")),
+ Left(14000023L, (2, 4L, "Hello world")),
+ Left(14000023L, (2, 5L, "Hello world")),
+ Right(14000030L)
+ )
+ val table = env
+ .addSource(new RowTimeSourceFunction[(Int, Long, String)](data))
+ .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+ val countFun = new CountAggFunction
+ val weightAvgFun = new WeightedAvg
+ val plusOne = new JavaFunc0
+
+ val windowedTable = table
+ .window(Over partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE following
+ CURRENT_RANGE as 'w)
+ .select(
+ 'a, 'b, 'c,
+ 'b.sum over 'w,
+ "SUM:".toExpr + ('b.sum over 'w),
+ countFun('b) over 'w,
+ (countFun('b) over 'w) + 1,
+ plusOne(countFun('b) over 'w),
+ array('b.avg over 'w, 'b.max over 'w),
+ 'b.avg over 'w,
+ 'b.max over 'w,
+ 'b.min over 'w,
+ ('b.min over 'w).abs(),
+ weightAvgFun('b, 'a) over 'w)
+
+ val result = windowedTable.toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "1,1,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
+ "1,2,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
+ "1,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
+ "1,1,Hi,7,SUM:7,4,5,5,[1, 3],1,3,1,1,1",
+ "2,1,Hello,1,SUM:1,1,2,2,[1, 1],1,1,1,1,1",
+ "2,2,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
+ "2,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
+ "1,4,Hello world,11,SUM:11,5,6,6,[2, 4],2,4,1,1,2",
+ "1,5,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
+ "1,6,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
+ "1,7,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
+ "2,4,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3",
+ "2,5,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3"
+ )
+
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testProcTimeBoundedPartitionedRowsOver(): Unit = {
+
+ val data = List(
+ (1, 1L, 0, "Hallo", 1L),
+ (2, 2L, 1, "Hallo Welt", 2L),
+ (2, 3L, 2, "Hallo Welt wie", 1L),
+ (3, 4L, 3, "Hallo Welt wie gehts?", 2L),
+ (3, 5L, 4, "ABC", 2L),
+ (3, 6L, 5, "BCD", 3L),
+ (4, 7L, 6, "CDE", 2L),
+ (4, 8L, 7, "DEF", 1L),
+ (4, 9L, 8, "EFG", 1L),
+ (4, 10L, 9, "FGH", 2L),
+ (5, 11L, 10, "GHI", 1L),
+ (5, 12L, 11, "HIJ", 3L),
+ (5, 13L, 12, "IJK", 3L),
+ (5, 14L, 13, "JKL", 2L),
+ (5, 15L, 14, "KLM", 2L))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStateBackend(getStateBackend)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ env.setParallelism(1)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
+
+ val windowedTable = table
+ .window(Over partitionBy 'a orderBy 'proctime preceding 4.rows following CURRENT_ROW as 'w)
+ .select('a, 'c.sum over 'w, 'c.min over 'w)
+ val result = windowedTable.toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "1,0,0",
+ "2,1,1",
+ "2,3,1",
+ "3,3,3",
+ "3,7,3",
+ "3,12,3",
+ "4,6,6",
+ "4,13,6",
+ "4,21,6",
+ "4,30,6",
+ "5,10,10",
+ "5,21,10",
+ "5,33,10",
+ "5,46,10",
+ "5,60,10")
+
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testRowTimeBoundedPartitionedRowOver(): Unit = {
+ val data = Seq(
+ Left((1L, (1L, 1, "Hello"))),
+ Left((2L, (2L, 2, "Hello"))),
+ Left((1L, (1L, 1, "Hello"))),
+ Left((2L, (2L, 2, "Hello"))),
+ Left((2L, (2L, 2, "Hello"))),
+ Left((1L, (1L, 1, "Hello"))),
+ Left((3L, (7L, 7, "Hello World"))),
+ Left((1L, (7L, 7, "Hello World"))),
+ Left((1L, (7L, 7, "Hello World"))),
+ Right(2L),
+ Left((3L, (3L, 3, "Hello"))),
+ Left((4L, (4L, 4, "Hello"))),
+ Left((5L, (5L, 5, "Hello"))),
+ Left((6L, (6L, 6, "Hello"))),
+ Left((20L, (20L, 20, "Hello World"))),
+ Right(6L),
+ Left((8L, (8L, 8, "Hello World"))),
+ Left((7L, (7L, 7, "Hello World"))),
+ Right(20L))
+
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ env.setStateBackend(getStateBackend)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+
+ val table = env.addSource[(Long, Int, String)](
+ new RowTimeSourceFunction[(Long, Int, String)](data))
+ .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+ val windowedTable = table
+ .window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
+ .select('c, 'a, 'a.count over 'w, 'a.sum over 'w)
+
+ val result = windowedTable.toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
+ "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
+ "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
+ "Hello,6,3,15",
+ "Hello World,7,1,7", "Hello World,7,2,14", "Hello World,7,3,21",
+ "Hello World,7,3,21", "Hello World,8,3,22", "Hello World,20,3,35")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testRowTimeBoundedPartitionedRangeOver(): Unit = {
+ val data = Seq(
+ Left((1500L, (1L, 15, "Hello"))),
+ Left((1600L, (1L, 16, "Hello"))),
+ Left((1000L, (1L, 1, "Hello"))),
+ Left((2000L, (2L, 2, "Hello"))),
+ Right(1000L),
+ Left((2000L, (2L, 2, "Hello"))),
+ Left((2000L, (2L, 3, "Hello"))),
+ Left((3000L, (3L, 3, "Hello"))),
+ Right(2000L),
+ Left((4000L, (4L, 4, "Hello"))),
+ Right(3000L),
+ Left((5000L, (5L, 5, "Hello"))),
+ Right(5000L),
+ Left((6000L, (6L, 6, "Hello"))),
+ Left((6500L, (6L, 65, "Hello"))),
+ Right(7000L),
+ Left((9000L, (6L, 9, "Hello"))),
+ Left((9500L, (6L, 18, "Hello"))),
+ Left((9000L, (6L, 9, "Hello"))),
+ Right(10000L),
+ Left((10000L, (7L, 7, "Hello World"))),
+ Left((11000L, (7L, 17, "Hello World"))),
+ Left((11000L, (7L, 77, "Hello World"))),
+ Right(12000L),
+ Left((14000L, (7L, 18, "Hello World"))),
+ Right(14000L),
+ Left((15000L, (8L, 8, "Hello World"))),
+ Right(17000L),
+ Left((20000L, (20L, 20, "Hello World"))),
+ Right(19000L))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ env.setStateBackend(getStateBackend)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+
+ val table = env.addSource[(Long, Int, String)](
+ new RowTimeSourceFunction[(Long, Int, String)](data))
+ .toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+
+ val windowedTable = table
+ .window(
+ Over partitionBy 'c orderBy 'rowtime preceding 1.seconds following CURRENT_RANGE as 'w)
+ .select('c, 'b, 'a.count over 'w, 'a.sum over 'w)
+
+ val result = windowedTable.toAppendStream[Row]
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
+ "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
+ "Hello,3,4,9",
+ "Hello,4,2,7",
+ "Hello,5,2,9",
+ "Hello,6,2,11", "Hello,65,2,12",
+ "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
+ "Hello World,7,1,7", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
+ "Hello World,8,2,15",
+ "Hello World,20,1,20")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
+
+object OverWindowITCase {
+
+ class RowTimeSourceFunction[T](
+ dataWithTimestampList: Seq[Either[(Long, T), Long]]) extends SourceFunction[T] {
+ override def run(ctx: SourceContext[T]): Unit = {
+ dataWithTimestampList.foreach {
+ case Left(t) => ctx.collectWithTimestamp(t._2, t._1)
+ case Right(w) => ctx.emitWatermark(new Watermark(w))
+ }
+ }
+
+ override def cancel(): Unit = ???
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala
new file mode 100644
index 0000000..7e948f4
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/RetractionITCase.scala
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.runtime.utils.{StreamITCase, StreamingWithStateTestBase}
+import org.apache.flink.table.utils.TableFunc0
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+/**
+ * tests for retraction
+ */
+class RetractionITCase extends StreamingWithStateTestBase {
+ // input data
+ val data = List(
+ ("Hello", 1),
+ ("word", 1),
+ ("Hello", 1),
+ ("bark", 1),
+ ("bark", 1),
+ ("bark", 1),
+ ("bark", 1),
+ ("bark", 1),
+ ("bark", 1),
+ ("flink", 1)
+ )
+
+ // keyed groupby + keyed groupby
+ @Test
+ def testWordCount(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+ env.setStateBackend(getStateBackend)
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'word, 'num)
+ val resultTable = table
+ .groupBy('word)
+ .select('num.sum as 'count)
+ .groupBy('count)
+ .select('count, 'count.count as 'frequency)
+
+ val results = resultTable.toRetractStream[Row]
+ results.addSink(new StreamITCase.RetractingSink)
+ env.execute()
+
+ val expected = Seq("1,2", "2,1", "6,1")
+ assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
+ }
+
+ // keyed groupby + non-keyed groupby
+ @Test
+ def testGroupByAndNonKeyedGroupBy(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+ env.setStateBackend(getStateBackend)
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'word, 'num)
+ val resultTable = table
+ .groupBy('word)
+ .select('word as 'word, 'num.sum as 'cnt)
+ .select('cnt.sum)
+
+ val results = resultTable.toRetractStream[Row]
+
+ results.addSink(new StreamITCase.RetractingSink).setParallelism(1)
+ env.execute()
+
+ val expected = Seq("10")
+ assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
+ }
+
+ // non-keyed groupby + keyed groupby
+ @Test
+ def testNonKeyedGroupByAndGroupBy(): Unit = {
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+ env.setStateBackend(getStateBackend)
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'word, 'num)
+ val resultTable = table
+ .select('num.sum as 'count)
+ .groupBy('count)
+ .select('count, 'count.count)
+
+ val results = resultTable.toRetractStream[Row]
+ results.addSink(new StreamITCase.RetractingSink)
+ env.execute()
+
+ val expected = Seq("10,1")
+ assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
+ }
+
+ // test unique process, if the current output message of unbounded groupby equals the
+ // previous message, unbounded groupby will ignore the current one.
+ @Test
+ def testUniqueProcess(): Unit = {
+ // data input
+ val data = List(
+ (1, 1L),
+ (2, 2L),
+ (3, 3L),
+ (3, 3L),
+ (4, 1L),
+ (4, 0L),
+ (4, 0L),
+ (4, 0L),
+ (5, 1L),
+ (6, 6L),
+ (6, 6L),
+ (6, 6L),
+ (7, 8L)
+ )
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+ env.setStateBackend(getStateBackend)
+ env.setParallelism(1)
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'pk, 'value)
+ val resultTable = table
+ .groupBy('pk)
+ .select('pk as 'pk, 'value.sum as 'sum)
+ .groupBy('sum)
+ .select('sum, 'pk.count as 'count)
+
+ val results = resultTable.toRetractStream[Row]
+ results.addSink(new StreamITCase.RetractMessagesSink)
+ env.execute()
+
+ val expected = Seq(
+ "+1,1", "+2,1", "+3,1", "-3,1", "+6,1", "-1,1", "+1,2", "-1,2", "+1,3", "-6,1", "+6,2",
+ "-6,2", "+6,1", "+12,1", "-12,1", "+18,1", "+8,1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ // correlate should handle retraction messages correctly
+ @Test
+ def testCorrelate(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+ env.setStateBackend(getStateBackend)
+
+ val func0 = new TableFunc0
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'word, 'num)
+ val resultTable = table
+ .groupBy('word)
+ .select('word as 'word, 'num.sum as 'cnt)
+ .leftOuterJoin(func0('word))
+ .groupBy('cnt)
+ .select('cnt, 'word.count as 'frequency)
+
+ val results = resultTable.toRetractStream[Row]
+ results.addSink(new StreamITCase.RetractingSink)
+ env.execute()
+
+ val expected = Seq("1,2", "2,1", "6,1")
+ assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala
new file mode 100644
index 0000000..688849e
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/SetOperatorsITCase.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData}
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class SetOperatorsITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test
+ def testUnion(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ StreamITCase.testResults = mutable.MutableList()
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'd, 'e, 'f)
+
+ val unionDs = ds1.unionAll(ds2).select('c)
+
+ val results = unionDs.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "Hi", "Hello", "Hello world", "Hi", "Hello", "Hello world")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testUnionWithFilter(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ StreamITCase.testResults = mutable.MutableList()
+ val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
+ val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
+
+ val unionDs = ds1.unionAll(ds2.select('a, 'b, 'c)).filter('b < 2).select('c)
+
+ val results = unionDs.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = mutable.MutableList("Hi", "Hallo")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala
new file mode 100644
index 0000000..4121754
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSinkITCase.scala
@@ -0,0 +1,509 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import java.io.File
+import java.lang.{Boolean => JBool}
+
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.datastream.DataStream
+import org.apache.flink.streaming.api.functions.sink.SinkFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.runtime.utils.StreamTestData
+import org.apache.flink.table.sinks._
+import org.apache.flink.test.util.TestBaseUtils
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class TableSinkITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test
+ def testStreamTableSink(): Unit = {
+
+ val tmpFile = File.createTempFile("flink-table-sink-test", ".tmp")
+ tmpFile.deleteOnExit()
+ val path = tmpFile.toURI.toString
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ env.setParallelism(4)
+
+ val input = StreamTestData.get3TupleDataStream(env)
+ .map(x => x).setParallelism(4) // increase DOP to 4
+
+ val results = input.toTable(tEnv, 'a, 'b, 'c)
+ .where('a < 5 || 'a > 17)
+ .select('c, 'b)
+ .writeToSink(new CsvTableSink(path))
+
+ env.execute()
+
+ val expected = Seq(
+ "Hi,1", "Hello,2", "Hello world,2", "Hello world, how are you?,3",
+ "Comment#12,6", "Comment#13,6", "Comment#14,6", "Comment#15,6").mkString("\n")
+
+ TestBaseUtils.compareResultsByLinesInMemory(expected, path)
+ }
+
+ @Test
+ def testAppendSinkOnAppendTable(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w)
+ .select('w.end, 'id.count, 'num.sum)
+ .writeToSink(new TestAppendSink)
+
+ env.execute()
+
+ val result = RowCollector.getAndClearValues.map(_.f1.toString).sorted
+ val expected = List(
+ "1970-01-01 00:00:00.005,4,8",
+ "1970-01-01 00:00:00.01,5,18",
+ "1970-01-01 00:00:00.015,5,24",
+ "1970-01-01 00:00:00.02,5,29",
+ "1970-01-01 00:00:00.025,2,12")
+ .sorted
+ assertEquals(expected, result)
+ }
+
+ @Test
+ def testRetractSinkOnUpdatingTable(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text)
+
+ t.select('id, 'num, 'text.charLength() as 'len)
+ .groupBy('len)
+ .select('len, 'id.count, 'num.sum)
+ .writeToSink(new TestRetractSink)
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ val retracted = restractResults(results).sorted
+ val expected = List(
+ "2,1,1",
+ "5,1,2",
+ "11,1,2",
+ "25,1,3",
+ "10,7,39",
+ "14,1,3",
+ "9,9,41").sorted
+ assertEquals(expected, retracted)
+
+ }
+
+ @Test
+ def testRetractSinkOnAppendTable(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w)
+ .select('w.end, 'id.count, 'num.sum)
+ .writeToSink(new TestRetractSink)
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertFalse(
+ "Received retraction messages for append only table",
+ results.exists(!_.f0))
+
+ val retracted = restractResults(results).sorted
+ val expected = List(
+ "1970-01-01 00:00:00.005,4,8",
+ "1970-01-01 00:00:00.01,5,18",
+ "1970-01-01 00:00:00.015,5,24",
+ "1970-01-01 00:00:00.02,5,29",
+ "1970-01-01 00:00:00.025,2,12")
+ .sorted
+ assertEquals(expected, retracted)
+
+ }
+
+ @Test
+ def testUpsertSinkOnUpdatingTableWithFullKey(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text)
+
+ t.select('id, 'num, 'text.charLength() as 'len, ('id > 0) as 'cTrue)
+ .groupBy('len, 'cTrue)
+ .select('len, 'id.count as 'cnt, 'cTrue)
+ .groupBy('cnt, 'cTrue)
+ .select('cnt, 'len.count, 'cTrue)
+ .writeToSink(new TestUpsertSink(Array("cnt", "cTrue"), false))
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertTrue(
+ "Results must include delete messages",
+ results.exists(_.f0 == false)
+ )
+
+ val retracted = upsertResults(results, Array(0, 2)).sorted
+ val expected = List(
+ "1,5,true",
+ "7,1,true",
+ "9,1,true").sorted
+ assertEquals(expected, retracted)
+
+ }
+
+
+
+ @Test
+ def testUpsertSinkOnAppendingTableWithFullKey1(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w, 'num)
+ .select('num, 'w.end as 'wend, 'id.count)
+ .writeToSink(new TestUpsertSink(Array("wend", "num"), true))
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertFalse(
+ "Received retraction messages for append only table",
+ results.exists(!_.f0))
+
+ val retracted = upsertResults(results, Array(0, 1, 2)).sorted
+ val expected = List(
+ "1,1970-01-01 00:00:00.005,1",
+ "2,1970-01-01 00:00:00.005,2",
+ "3,1970-01-01 00:00:00.005,1",
+ "3,1970-01-01 00:00:00.01,2",
+ "4,1970-01-01 00:00:00.01,3",
+ "4,1970-01-01 00:00:00.015,1",
+ "5,1970-01-01 00:00:00.015,4",
+ "5,1970-01-01 00:00:00.02,1",
+ "6,1970-01-01 00:00:00.02,4",
+ "6,1970-01-01 00:00:00.025,2").sorted
+ assertEquals(expected, retracted)
+ }
+
+ @Test
+ def testUpsertSinkOnAppendingTableWithFullKey2(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w, 'num)
+ .select('w.start as 'wstart, 'w.end as 'wend, 'num, 'id.count)
+ .writeToSink(new TestUpsertSink(Array("wstart", "wend", "num"), true))
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertFalse(
+ "Received retraction messages for append only table",
+ results.exists(!_.f0))
+
+ val retracted = upsertResults(results, Array(0, 1, 2)).sorted
+ val expected = List(
+ "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,1,1",
+ "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,2,2",
+ "1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,3,1",
+ "1970-01-01 00:00:00.005,1970-01-01 00:00:00.01,3,2",
+ "1970-01-01 00:00:00.005,1970-01-01 00:00:00.01,4,3",
+ "1970-01-01 00:00:00.01,1970-01-01 00:00:00.015,4,1",
+ "1970-01-01 00:00:00.01,1970-01-01 00:00:00.015,5,4",
+ "1970-01-01 00:00:00.015,1970-01-01 00:00:00.02,5,1",
+ "1970-01-01 00:00:00.015,1970-01-01 00:00:00.02,6,4",
+ "1970-01-01 00:00:00.02,1970-01-01 00:00:00.025,6,2").sorted
+ assertEquals(expected, retracted)
+ }
+
+ @Test
+ def testUpsertSinkOnAppendingTableWithoutFullKey1(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w, 'num)
+ .select('w.end as 'wend, 'id.count as 'cnt)
+ .writeToSink(new TestUpsertSink(null, true))
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertFalse(
+ "Received retraction messages for append only table",
+ results.exists(!_.f0))
+
+ val retracted = results.map(_.f1.toString).sorted
+ val expected = List(
+ "1970-01-01 00:00:00.005,1",
+ "1970-01-01 00:00:00.005,2",
+ "1970-01-01 00:00:00.005,1",
+ "1970-01-01 00:00:00.01,2",
+ "1970-01-01 00:00:00.01,3",
+ "1970-01-01 00:00:00.015,1",
+ "1970-01-01 00:00:00.015,4",
+ "1970-01-01 00:00:00.02,1",
+ "1970-01-01 00:00:00.02,4",
+ "1970-01-01 00:00:00.025,2").sorted
+ assertEquals(expected, retracted)
+ }
+
+ @Test
+ def testUpsertSinkOnAppendingTableWithoutFullKey2(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ val t = StreamTestData.get3TupleDataStream(env)
+ .assignAscendingTimestamps(_._1.toLong)
+ .toTable(tEnv, 'id, 'num, 'text, 'rowtime.rowtime)
+
+ t.window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w, 'num)
+ .select('num, 'id.count as 'cnt)
+ .writeToSink(new TestUpsertSink(null, true))
+
+ env.execute()
+ val results = RowCollector.getAndClearValues
+
+ assertFalse(
+ "Received retraction messages for append only table",
+ results.exists(!_.f0))
+
+ val retracted = results.map(_.f1.toString).sorted
+ val expected = List(
+ "1,1",
+ "2,2",
+ "3,1",
+ "3,2",
+ "4,3",
+ "4,1",
+ "5,4",
+ "5,1",
+ "6,4",
+ "6,2").sorted
+ assertEquals(expected, retracted)
+ }
+
+ /** Converts a list of retraction messages into a list of final results. */
+ private def restractResults(results: List[JTuple2[JBool, Row]]): List[String] = {
+
+ val retracted = results
+ .foldLeft(Map[String, Int]()){ (m: Map[String, Int], v: JTuple2[JBool, Row]) =>
+ val cnt = m.getOrElse(v.f1.toString, 0)
+ if (v.f0) {
+ m + (v.f1.toString -> (cnt + 1))
+ } else {
+ m + (v.f1.toString -> (cnt - 1))
+ }
+ }.filter{ case (_, c: Int) => c != 0 }
+
+ assertFalse(
+ "Received retracted rows which have not been accumulated.",
+ retracted.exists{ case (_, c: Int) => c < 0})
+
+ retracted.flatMap { case (r: String, c: Int) => (0 until c).map(_ => r) }.toList
+ }
+
+ /** Converts a list of upsert messages into a list of final results. */
+ private def upsertResults(results: List[JTuple2[JBool, Row]], keys: Array[Int]): List[String] = {
+
+ def getKeys(r: Row): List[String] =
+ keys.foldLeft(List[String]())((k, i) => r.getField(i).toString :: k)
+
+ val upserted = results.foldLeft(Map[String, String]()){ (o: Map[String, String], r) =>
+ val key = getKeys(r.f1).mkString("")
+ if (r.f0) {
+ o + (key -> r.f1.toString)
+ } else {
+ o - key
+ }
+ }
+
+ upserted.values.toList
+ }
+
+}
+
+private[flink] class TestAppendSink extends AppendStreamTableSink[Row] {
+
+ var fNames: Array[String] = _
+ var fTypes: Array[TypeInformation[_]] = _
+
+ override def emitDataStream(s: DataStream[Row]): Unit = {
+ s.map(
+ new MapFunction[Row, JTuple2[JBool, Row]] {
+ override def map(value: Row): JTuple2[JBool, Row] = new JTuple2(true, value)
+ })
+ .addSink(new RowSink)
+ }
+
+ override def getOutputType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+ override def getFieldNames: Array[String] = fNames
+
+ override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+ override def configure(
+ fieldNames: Array[String],
+ fieldTypes: Array[TypeInformation[_]]): TableSink[Row] = {
+ val copy = new TestAppendSink
+ copy.fNames = fieldNames
+ copy.fTypes = fieldTypes
+ copy
+ }
+}
+
+private[flink] class TestRetractSink extends RetractStreamTableSink[Row] {
+
+ var fNames: Array[String] = _
+ var fTypes: Array[TypeInformation[_]] = _
+
+ override def emitDataStream(s: DataStream[JTuple2[JBool, Row]]): Unit = {
+ s.addSink(new RowSink)
+ }
+
+ override def getRecordType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+ override def getFieldNames: Array[String] = fNames
+
+ override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+ override def configure(
+ fieldNames: Array[String],
+ fieldTypes: Array[TypeInformation[_]]): TableSink[JTuple2[JBool, Row]] = {
+ val copy = new TestRetractSink
+ copy.fNames = fieldNames
+ copy.fTypes = fieldTypes
+ copy
+ }
+
+}
+
+private[flink] class TestUpsertSink(
+ expectedKeys: Array[String],
+ expectedIsAppendOnly: Boolean)
+ extends UpsertStreamTableSink[Row] {
+
+ var fNames: Array[String] = _
+ var fTypes: Array[TypeInformation[_]] = _
+
+ override def setKeyFields(keys: Array[String]): Unit =
+ if (keys != null) {
+ assertEquals("Provided key fields do not match expected keys",
+ expectedKeys.sorted.mkString(","),
+ keys.sorted.mkString(","))
+ } else {
+ assertNull("Provided key fields should not be null.", expectedKeys)
+ }
+
+ override def setIsAppendOnly(isAppendOnly: JBool): Unit =
+ assertEquals(
+ "Provided isAppendOnly does not match expected isAppendOnly",
+ expectedIsAppendOnly,
+ isAppendOnly)
+
+ override def getRecordType: TypeInformation[Row] = new RowTypeInfo(fTypes, fNames)
+
+ override def emitDataStream(s: DataStream[JTuple2[JBool, Row]]): Unit = {
+ s.addSink(new RowSink)
+ }
+
+ override def getFieldNames: Array[String] = fNames
+
+ override def getFieldTypes: Array[TypeInformation[_]] = fTypes
+
+ override def configure(
+ fieldNames: Array[String],
+ fieldTypes: Array[TypeInformation[_]]): TableSink[JTuple2[JBool, Row]] = {
+ val copy = new TestUpsertSink(expectedKeys, expectedIsAppendOnly)
+ copy.fNames = fieldNames
+ copy.fTypes = fieldTypes
+ copy
+ }
+}
+
+class RowSink extends SinkFunction[JTuple2[JBool, Row]] {
+ override def invoke(value: JTuple2[JBool, Row]): Unit = RowCollector.addValue(value)
+}
+
+object RowCollector {
+ private val sink: mutable.ArrayBuffer[JTuple2[JBool, Row]] =
+ new mutable.ArrayBuffer[JTuple2[JBool, Row]]()
+
+ def addValue(value: JTuple2[JBool, Row]): Unit = {
+ sink.synchronized {
+ sink += value
+ }
+ }
+
+ def getAndClearValues: List[JTuple2[JBool, Row]] = {
+ val out = sink.toList
+ sink.clear()
+ out
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala
new file mode 100644
index 0000000..297e9cf
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/TableSourceITCase.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.api.TableEnvironment
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.runtime.utils.{CommonTestData, StreamITCase}
+import org.apache.flink.table.utils.TestFilterableTableSource
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+class TableSourceITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test
+ def testCsvTableSource(): Unit = {
+
+ val csvTable = CommonTestData.getCsvTableSource
+ StreamITCase.testResults = mutable.MutableList()
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+
+ tEnv.registerTableSource("csvTable", csvTable)
+ tEnv.scan("csvTable")
+ .where('id > 4)
+ .select('last, 'score * 2)
+ .toAppendStream[Row]
+ .addSink(new StreamITCase.StringSink[Row])
+
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "Williams,69.0",
+ "Miller,13.56",
+ "Smith,180.2",
+ "Williams,4.68")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testCsvTableSourceWithFilterable(): Unit = {
+ StreamITCase.testResults = mutable.MutableList()
+ val tableName = "MyTable"
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ tEnv.registerTableSource(tableName, new TestFilterableTableSource)
+ tEnv.scan(tableName)
+ .where("amount > 4 && price < 9")
+ .select("id, name")
+ .addSink(new StreamITCase.StringSink[Row])
+
+ env.execute()
+
+ val expected = mutable.MutableList(
+ "5,Record_5", "6,Record_6", "7,Record_7", "8,Record_8")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala
new file mode 100644
index 0000000..d716d5d
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/CommonTestData.scala
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+import java.io.{File, FileOutputStream, OutputStreamWriter}
+import java.util
+
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.java.{DataSet, ExecutionEnvironment}
+import org.apache.flink.table.api.TableSchema
+import org.apache.flink.table.catalog._
+import org.apache.flink.table.sources.{BatchTableSource, CsvTableSource}
+
+object CommonTestData {
+
+ def getCsvTableSource: CsvTableSource = {
+ val csvRecords = Seq(
+ "First#Id#Score#Last",
+ "Mike#1#12.3#Smith",
+ "Bob#2#45.6#Taylor",
+ "Sam#3#7.89#Miller",
+ "Peter#4#0.12#Smith",
+ "% Just a comment",
+ "Liz#5#34.5#Williams",
+ "Sally#6#6.78#Miller",
+ "Alice#7#90.1#Smith",
+ "Kelly#8#2.34#Williams"
+ )
+
+ val tempFilePath = writeToTempFile(csvRecords.mkString("$"), "csv-test", "tmp")
+ new CsvTableSource(
+ tempFilePath,
+ Array("first", "id", "score", "last"),
+ Array(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.DOUBLE_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO
+ ),
+ fieldDelim = "#",
+ rowDelim = "$",
+ ignoreFirstLine = true,
+ ignoreComments = "%"
+ )
+ }
+
+ def getInMemoryTestCatalog: ExternalCatalog = {
+ val csvRecord1 = Seq(
+ "1#1#Hi",
+ "2#2#Hello",
+ "3#2#Hello world"
+ )
+ val tempFilePath1 = writeToTempFile(csvRecord1.mkString("$"), "csv-test1", "tmp")
+ val properties1 = new util.HashMap[String, String]()
+ properties1.put("path", tempFilePath1)
+ properties1.put("fieldDelim", "#")
+ properties1.put("rowDelim", "$")
+ val externalCatalogTable1 = ExternalCatalogTable(
+ "csv",
+ new TableSchema(
+ Array("a", "b", "c"),
+ Array(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO)),
+ properties1
+ )
+
+ val csvRecord2 = Seq(
+ "1#1#0#Hallo#1",
+ "2#2#1#Hallo Welt#2",
+ "2#3#2#Hallo Welt wie#1",
+ "3#4#3#Hallo Welt wie gehts?#2",
+ "3#5#4#ABC#2",
+ "3#6#5#BCD#3",
+ "4#7#6#CDE#2",
+ "4#8#7#DEF#1",
+ "4#9#8#EFG#1",
+ "4#10#9#FGH#2",
+ "5#11#10#GHI#1",
+ "5#12#11#HIJ#3",
+ "5#13#12#IJK#3",
+ "5#14#13#JKL#2",
+ "5#15#14#KLM#2"
+ )
+ val tempFilePath2 = writeToTempFile(csvRecord2.mkString("$"), "csv-test2", "tmp")
+ val properties2 = new util.HashMap[String, String]()
+ properties2.put("path", tempFilePath2)
+ properties2.put("fieldDelim", "#")
+ properties2.put("rowDelim", "$")
+ val externalCatalogTable2 = ExternalCatalogTable(
+ "csv",
+ new TableSchema(
+ Array("d", "e", "f", "g", "h"),
+ Array(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO)
+ ),
+ properties2
+ )
+ val catalog = new InMemoryExternalCatalog("test")
+ val db1 = new InMemoryExternalCatalog("db1")
+ val db2 = new InMemoryExternalCatalog("db2")
+ catalog.createSubCatalog("db1", db1, ignoreIfExists = false)
+ catalog.createSubCatalog("db2", db2, ignoreIfExists = false)
+
+ // Register the table with both catalogs
+ catalog.createTable("tb1", externalCatalogTable1, ignoreIfExists = false)
+ db1.createTable("tb1", externalCatalogTable1, ignoreIfExists = false)
+ db2.createTable("tb2", externalCatalogTable2, ignoreIfExists = false)
+ catalog
+ }
+
+ private def writeToTempFile(
+ contents: String,
+ filePrefix: String,
+ fileSuffix: String,
+ charset: String = "UTF-8"): String = {
+ val tempFile = File.createTempFile(filePrefix, fileSuffix)
+ tempFile.deleteOnExit()
+ val tmpWriter = new OutputStreamWriter(new FileOutputStream(tempFile), charset)
+ tmpWriter.write(contents)
+ tmpWriter.close()
+ tempFile.getAbsolutePath
+ }
+
+ def getNestedTableSource: BatchTableSource[Person] = {
+ new BatchTableSource[Person] {
+ override def getDataSet(execEnv: ExecutionEnvironment): DataSet[Person] = {
+ val executionEnvironment = ExecutionEnvironment.getExecutionEnvironment
+ executionEnvironment.fromCollection(
+ util.Arrays.asList(
+ new Person("Mike", "Smith", new Address("5th Ave", "New-York")),
+ new Person("Sally", "Miller", new Address("Potsdamer Platz", "Berlin")),
+ new Person("Bob", "Taylor", new Address("Pearse Street", "Dublin"))),
+ getReturnType
+ )
+ }
+
+ override def getReturnType: TypeInformation[Person] = {
+ TypeExtractor.getForClass(classOf[Person])
+ }
+ }
+ }
+
+ class Person(var firstName: String, var lastName: String, var address: Address) {
+ def this() {
+ this(null, null, null)
+ }
+ }
+
+ class Address(var street: String, var city: String) {
+ def this() {
+ this(null, null)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/SortTestUtils.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/SortTestUtils.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/SortTestUtils.scala
new file mode 100644
index 0000000..f102f53
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/SortTestUtils.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+object SortTestUtils {
+
+ val tupleDataSetStrings = List((1, 1L, "Hi")
+ ,(2, 2L, "Hello")
+ ,(3, 2L, "Hello world")
+ ,(4, 3L, "Hello world, how are you?")
+ ,(5, 3L, "I am fine.")
+ ,(6, 3L, "Luke Skywalker")
+ ,(7, 4L, "Comment#1")
+ ,(8, 4L, "Comment#2")
+ ,(9, 4L, "Comment#3")
+ ,(10, 4L, "Comment#4")
+ ,(11, 5L, "Comment#5")
+ ,(12, 5L, "Comment#6")
+ ,(13, 5L, "Comment#7")
+ ,(14, 5L, "Comment#8")
+ ,(15, 5L, "Comment#9")
+ ,(16, 6L, "Comment#10")
+ ,(17, 6L, "Comment#11")
+ ,(18, 6L, "Comment#12")
+ ,(19, 6L, "Comment#13")
+ ,(20, 6L, "Comment#14")
+ ,(21, 6L, "Comment#15"))
+
+ def sortExpectedly(dataSet: List[Product])
+ (implicit ordering: Ordering[Product]): String =
+ sortExpectedly(dataSet, 0, dataSet.length)
+
+ def sortExpectedly(dataSet: List[Product], start: Int, end: Int)
+ (implicit ordering: Ordering[Product]): String = {
+ dataSet
+ .sorted(ordering)
+ .slice(start, end)
+ .mkString("\n")
+ .replaceAll("[\\(\\)]", "")
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamITCase.scala
new file mode 100644
index 0000000..5386b40
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamITCase.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+import java.util.Collections
+
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
+import org.apache.flink.types.Row
+import org.junit.Assert._
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
+
+object StreamITCase {
+
+ var testResults: mutable.MutableList[String] = mutable.MutableList.empty[String]
+ var retractedResults: ArrayBuffer[String] = mutable.ArrayBuffer.empty[String]
+
+ def clear = {
+ StreamITCase.testResults.clear()
+ StreamITCase.retractedResults.clear()
+ }
+
+ def compareWithList(expected: java.util.List[String]): Unit = {
+ Collections.sort(expected)
+ assertEquals(expected.asScala, StreamITCase.testResults.sorted)
+ }
+
+ final class StringSink[T] extends RichSinkFunction[T]() {
+ def invoke(value: T) {
+ testResults.synchronized {
+ testResults += value.toString
+ }
+ }
+ }
+
+ final class RetractMessagesSink extends RichSinkFunction[(Boolean, Row)]() {
+ def invoke(v: (Boolean, Row)) {
+ testResults.synchronized {
+ testResults += (if (v._1) "+" else "-") + v._2
+ }
+ }
+ }
+
+ final class RetractingSink() extends RichSinkFunction[(Boolean, Row)] {
+ def invoke(v: (Boolean, Row)) {
+ retractedResults.synchronized {
+ val value = v._2.toString
+ if (v._1) {
+ retractedResults += value
+ } else {
+ val idx = retractedResults.indexOf(value)
+ if (idx >= 0) {
+ retractedResults.remove(idx)
+ } else {
+ throw new RuntimeException("Tried to retract a value that wasn't added first. " +
+ "This is probably an incorrectly implemented test. " +
+ "Try to set the parallelism of the sink to 1.")
+ }
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala
new file mode 100644
index 0000000..94ced19
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamTestData.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
+
+import scala.collection.mutable
+
+object StreamTestData {
+
+ def getSmall3TupleDataStream(env: StreamExecutionEnvironment): DataStream[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ env.fromCollection(data)
+ }
+
+ def get3TupleDataStream(env: StreamExecutionEnvironment): DataStream[(Int, Long, String)] = {
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "Hi"))
+ data.+=((2, 2L, "Hello"))
+ data.+=((3, 2L, "Hello world"))
+ data.+=((4, 3L, "Hello world, how are you?"))
+ data.+=((5, 3L, "I am fine."))
+ data.+=((6, 3L, "Luke Skywalker"))
+ data.+=((7, 4L, "Comment#1"))
+ data.+=((8, 4L, "Comment#2"))
+ data.+=((9, 4L, "Comment#3"))
+ data.+=((10, 4L, "Comment#4"))
+ data.+=((11, 5L, "Comment#5"))
+ data.+=((12, 5L, "Comment#6"))
+ data.+=((13, 5L, "Comment#7"))
+ data.+=((14, 5L, "Comment#8"))
+ data.+=((15, 5L, "Comment#9"))
+ data.+=((16, 6L, "Comment#10"))
+ data.+=((17, 6L, "Comment#11"))
+ data.+=((18, 6L, "Comment#12"))
+ data.+=((19, 6L, "Comment#13"))
+ data.+=((20, 6L, "Comment#14"))
+ data.+=((21, 6L, "Comment#15"))
+ env.fromCollection(data)
+ }
+
+ def get5TupleDataStream(env: StreamExecutionEnvironment):
+ DataStream[(Int, Long, Int, String, Long)] = {
+
+ val data = new mutable.MutableList[(Int, Long, Int, String, Long)]
+ data.+=((1, 1L, 0, "Hallo", 1L))
+ data.+=((2, 2L, 1, "Hallo Welt", 2L))
+ data.+=((2, 3L, 2, "Hallo Welt wie", 1L))
+ data.+=((3, 4L, 3, "Hallo Welt wie gehts?", 2L))
+ data.+=((3, 5L, 4, "ABC", 2L))
+ data.+=((3, 6L, 5, "BCD", 3L))
+ data.+=((4, 7L, 6, "CDE", 2L))
+ data.+=((4, 8L, 7, "DEF", 1L))
+ data.+=((4, 9L, 8, "EFG", 1L))
+ data.+=((4, 10L, 9, "FGH", 2L))
+ data.+=((5, 11L, 10, "GHI", 1L))
+ data.+=((5, 12L, 11, "HIJ", 3L))
+ data.+=((5, 13L, 12, "IJK", 3L))
+ data.+=((5, 14L, 13, "JKL", 2L))
+ data.+=((5, 15L, 14, "KLM", 2L))
+ env.fromCollection(data)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala
new file mode 100644
index 0000000..8c41f22
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/StreamingWithStateTestBase.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.utils
+
+import org.apache.flink.contrib.streaming.state.RocksDBStateBackend
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
+
+class StreamingWithStateTestBase extends StreamingMultipleProgramsTestBase {
+
+ val _tempFolder = new TemporaryFolder
+
+ @Rule
+ def tempFolder: TemporaryFolder = _tempFolder
+
+ def getStateBackend: RocksDBStateBackend = {
+ val dbPath = tempFolder.newFolder().getAbsolutePath
+ val checkpointPath = tempFolder.newFolder().toURI.toString
+ val backend = new RocksDBStateBackend(checkpointPath)
+ backend.setDbStoragePath(dbPath)
+ backend
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsClusterTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsClusterTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsClusterTestBase.scala
new file mode 100644
index 0000000..77b58a7
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsClusterTestBase.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+import java.util
+
+import org.apache.flink.table.runtime.batch.table.GroupWindowITCase
+import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
+import org.junit.runners.Parameterized
+
+import scala.collection.JavaConversions._
+
+/**
+ * This test base provides full cluster-like integration tests for batch programs. Only runtime
+ * operator tests should use this test base as they are expensive.
+ * (e.g. [[GroupWindowITCase]])
+ */
+class TableProgramsClusterTestBase(
+ executionMode: TestExecutionMode,
+ tableConfigMode: TableConfigMode)
+ extends TableProgramsTestBase(executionMode, tableConfigMode) {
+}
+
+object TableProgramsClusterTestBase {
+
+ @Parameterized.Parameters(name = "Execution mode = {0}, Table config = {1}")
+ def parameters(): util.Collection[Array[java.lang.Object]] = {
+ Seq[Array[AnyRef]](
+ Array(TestExecutionMode.CLUSTER, TableProgramsTestBase.DEFAULT),
+ Array(TestExecutionMode.CLUSTER_OBJECT_REUSE, TableProgramsTestBase.DEFAULT)
+ )
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsCollectionTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsCollectionTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsCollectionTestBase.scala
new file mode 100644
index 0000000..8947855
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/utils/TableProgramsCollectionTestBase.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.utils
+
+import java.util
+
+import org.apache.flink.table.runtime.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
+import org.junit.runners.Parameterized
+
+import scala.collection.JavaConversions._
+
+/**
+ * This test base provides lightweight integration tests for batch programs. However, it does
+ * not test everything (e.g. combiners). Runtime operator tests should
+ * use [[TableProgramsClusterTestBase]].
+ */
+class TableProgramsCollectionTestBase(
+ tableConfigMode: TableConfigMode)
+ extends TableProgramsTestBase(TestExecutionMode.COLLECTION, tableConfigMode) {
+}
+
+object TableProgramsCollectionTestBase {
+
+ @Parameterized.Parameters(name = "Table config = {0}")
+ def parameters(): util.Collection[Array[java.lang.Object]] = {
+ Seq[Array[AnyRef]](Array(TableProgramsTestBase.DEFAULT))
+ }
+}