You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2017/07/13 10:18:30 UTC
[21/44] flink git commit: [FLINK-6617] [table] Restructuring of tests
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/TimeAttributesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/TimeAttributesITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/TimeAttributesITCase.scala
deleted file mode 100644
index 2f9549e..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/TimeAttributesITCase.scala
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.runtime.datastream.table
-
-import java.math.BigDecimal
-
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.TimeCharacteristic
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import org.apache.flink.streaming.api.watermark.Watermark
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
-import org.apache.flink.table.api.java.utils.Pojos.Pojo1
-import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.{TableEnvironment, Types}
-import org.apache.flink.table.calcite.table.RelTimeIndicatorConverterTest.TableFunc
-import org.apache.flink.table.expressions.{ExpressionParser, TimeIntervalUnit}
-import org.apache.flink.table.runtime.datastream.StreamITCase
-import org.apache.flink.table.runtime.datastream.table.TimeAttributesITCase._
-import org.apache.flink.types.Row
-import org.junit.Assert._
-import org.junit.Test
-
-import scala.collection.mutable
-
-/**
- * Tests for access and materialization of time attributes.
- */
-class TimeAttributesITCase extends StreamingMultipleProgramsTestBase {
-
- val data = List(
- (1L, 1, 1d, 1f, new BigDecimal("1"), "Hi"),
- (2L, 2, 2d, 2f, new BigDecimal("2"), "Hallo"),
- (3L, 2, 2d, 2f, new BigDecimal("2"), "Hello"),
- (4L, 5, 5d, 5f, new BigDecimal("5"), "Hello"),
- (7L, 3, 3d, 3f, new BigDecimal("3"), "Hello"),
- (8L, 3, 3d, 3f, new BigDecimal("3"), "Hello world"),
- (16L, 4, 4d, 4f, new BigDecimal("4"), "Hello world"))
-
- @Test
- def testCalcMaterialization(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
-
- val t = table.select('rowtime.cast(Types.STRING))
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.001",
- "1970-01-01 00:00:00.002",
- "1970-01-01 00:00:00.003",
- "1970-01-01 00:00:00.004",
- "1970-01-01 00:00:00.007",
- "1970-01-01 00:00:00.008",
- "1970-01-01 00:00:00.016")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testCalcMaterialization2(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
-
- val t = table
- .filter('rowtime.cast(Types.LONG) > 4)
- .select('rowtime, 'rowtime.floor(TimeIntervalUnit.DAY), 'rowtime.ceil(TimeIntervalUnit.DAY))
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.007,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
- "1970-01-01 00:00:00.008,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
- "1970-01-01 00:00:00.016,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testTableFunction(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(
- tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string, 'proctime.proctime)
- val func = new TableFunc
-
- val t = table.join(func('rowtime, 'proctime, 'string) as 's).select('rowtime, 's)
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.001,1trueHi",
- "1970-01-01 00:00:00.002,2trueHallo",
- "1970-01-01 00:00:00.003,3trueHello",
- "1970-01-01 00:00:00.004,4trueHello",
- "1970-01-01 00:00:00.007,7trueHello",
- "1970-01-01 00:00:00.008,8trueHello world",
- "1970-01-01 00:00:00.016,16trueHello world")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testWindowAfterTableFunction(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(
- tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string, 'proctime.proctime)
- val func = new TableFunc
-
- val t = table
- .join(func('rowtime, 'proctime, 'string) as 's)
- .window(Tumble over 5.millis on 'rowtime as 'w)
- .groupBy('w)
- .select('w.rowtime, 's.count)
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.004,4",
- "1970-01-01 00:00:00.009,2",
- "1970-01-01 00:00:00.019,1")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testUnion(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(
- tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
-
- val t = table.unionAll(table).select('rowtime)
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.001",
- "1970-01-01 00:00:00.001",
- "1970-01-01 00:00:00.002",
- "1970-01-01 00:00:00.002",
- "1970-01-01 00:00:00.003",
- "1970-01-01 00:00:00.003",
- "1970-01-01 00:00:00.004",
- "1970-01-01 00:00:00.004",
- "1970-01-01 00:00:00.007",
- "1970-01-01 00:00:00.007",
- "1970-01-01 00:00:00.008",
- "1970-01-01 00:00:00.008",
- "1970-01-01 00:00:00.016",
- "1970-01-01 00:00:00.016")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testWindowWithAggregationOnRowtimeSql(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
- tEnv.registerTable("MyTable", table)
-
- val t = tEnv.sql("SELECT COUNT(`rowtime`) FROM MyTable " +
- "GROUP BY TUMBLE(rowtime, INTERVAL '0.003' SECOND)")
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1",
- "2",
- "2",
- "2"
- )
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testMultiWindow(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
-
- val t = table
- .window(Tumble over 2.millis on 'rowtime as 'w)
- .groupBy('w)
- .select('w.rowtime as 'rowtime, 'int.count as 'int)
- .window(Tumble over 4.millis on 'rowtime as 'w2)
- .groupBy('w2)
- .select('w2.rowtime, 'w2.end, 'int.count)
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.003,1970-01-01 00:00:00.004,2",
- "1970-01-01 00:00:00.007,1970-01-01 00:00:00.008,2",
- "1970-01-01 00:00:00.011,1970-01-01 00:00:00.012,1",
- "1970-01-01 00:00:00.019,1970-01-01 00:00:00.02,1"
- )
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testCalcMaterializationWithPojoType(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val stream = env
- .fromCollection(data)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
- val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
- tEnv.registerTable("T1", table)
- val querySql = "select rowtime as ts, string as msg from T1"
-
- val results = tEnv.sql(querySql).toAppendStream[Pojo1]
- results.addSink(new StreamITCase.StringSink[Pojo1])
- env.execute()
-
- val expected = Seq(
- "Pojo1{ts=1970-01-01 00:00:00.001, msg='Hi'}",
- "Pojo1{ts=1970-01-01 00:00:00.002, msg='Hallo'}",
- "Pojo1{ts=1970-01-01 00:00:00.003, msg='Hello'}",
- "Pojo1{ts=1970-01-01 00:00:00.004, msg='Hello'}",
- "Pojo1{ts=1970-01-01 00:00:00.007, msg='Hello'}",
- "Pojo1{ts=1970-01-01 00:00:00.008, msg='Hello world'}",
- "Pojo1{ts=1970-01-01 00:00:00.016, msg='Hello world'}")
-
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testPojoSupport(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamITCase.testResults = mutable.MutableList()
-
- val p1 = new TestPojo
- p1.a = 12
- p1.b = 42L
- p1.c = "Test me."
-
- val p2 = new TestPojo
- p2.a = 13
- p2.b = 43L
- p2.c = "And me."
-
- val stream = env
- .fromElements(p1, p2)
- .assignTimestampsAndWatermarks(new TimestampWithEqualWatermarkPojo)
- // use aliases, swap all attributes, and skip b2
- val table = stream.toTable(tEnv, ('b as 'b).rowtime, 'c as 'c, 'a as 'a)
- // no aliases, no swapping
- val table2 = stream.toTable(tEnv, 'a, 'b.rowtime, 'c)
- // use proctime, no skipping
- val table3 = stream.toTable(tEnv, 'a, 'b.rowtime, 'c, 'b2, 'proctime.proctime)
-
- // Java expressions
-
- // use aliases, swap all attributes, and skip b2
- val table4 = stream.toTable(
- tEnv,
- ExpressionParser.parseExpressionList("(b as b).rowtime, c as c, a as a"): _*)
- // no aliases, no swapping
- val table5 = stream.toTable(
- tEnv,
- ExpressionParser.parseExpressionList("a, b.rowtime, c"): _*)
-
- val t = table.select('b, 'c , 'a)
- .unionAll(table2.select('b, 'c, 'a))
- .unionAll(table3.select('b, 'c, 'a))
- .unionAll(table4.select('b, 'c, 'a))
- .unionAll(table5.select('b, 'c, 'a))
-
- val results = t.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = Seq(
- "1970-01-01 00:00:00.042,Test me.,12",
- "1970-01-01 00:00:00.042,Test me.,12",
- "1970-01-01 00:00:00.042,Test me.,12",
- "1970-01-01 00:00:00.042,Test me.,12",
- "1970-01-01 00:00:00.042,Test me.,12",
- "1970-01-01 00:00:00.043,And me.,13",
- "1970-01-01 00:00:00.043,And me.,13",
- "1970-01-01 00:00:00.043,And me.,13",
- "1970-01-01 00:00:00.043,And me.,13",
- "1970-01-01 00:00:00.043,And me.,13")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-}
-
-object TimeAttributesITCase {
- class TimestampWithEqualWatermark
- extends AssignerWithPunctuatedWatermarks[(Long, Int, Double, Float, BigDecimal, String)] {
-
- override def checkAndGetNextWatermark(
- lastElement: (Long, Int, Double, Float, BigDecimal, String),
- extractedTimestamp: Long)
- : Watermark = {
- new Watermark(extractedTimestamp)
- }
-
- override def extractTimestamp(
- element: (Long, Int, Double, Float, BigDecimal, String),
- previousElementTimestamp: Long): Long = {
- element._1
- }
- }
-
- class TimestampWithEqualWatermarkPojo
- extends AssignerWithPunctuatedWatermarks[TestPojo] {
-
- override def checkAndGetNextWatermark(
- lastElement: TestPojo,
- extractedTimestamp: Long)
- : Watermark = {
- new Watermark(extractedTimestamp)
- }
-
- override def extractTimestamp(
- element: TestPojo,
- previousElementTimestamp: Long): Long = {
- element.b
- }
- }
-
- class TestPojo() {
- var a: Int = _
- var b: Long = _
- var b2: String = "skip me"
- var c: String = _
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UnionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UnionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UnionITCase.scala
deleted file mode 100644
index f58b8a2..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UnionITCase.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.runtime.datastream.table
-
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
-import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.scala.stream.utils.StreamTestData
-import org.apache.flink.table.runtime.datastream.StreamITCase
-import org.apache.flink.types.Row
-import org.junit.Assert._
-import org.junit.Test
-
-import scala.collection.mutable
-
-class UnionITCase extends StreamingMultipleProgramsTestBase {
-
- @Test
- def testUnion(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- val tEnv = TableEnvironment.getTableEnvironment(env)
-
- StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
- val ds2 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'd, 'e, 'f)
-
- val unionDs = ds1.unionAll(ds2).select('c)
-
- val results = unionDs.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "Hi", "Hello", "Hello world", "Hi", "Hello", "Hello world")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testUnionWithFilter(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- val tEnv = TableEnvironment.getTableEnvironment(env)
-
- StreamITCase.testResults = mutable.MutableList()
- val ds1 = StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv, 'a, 'b, 'c)
- val ds2 = StreamTestData.get5TupleDataStream(env).toTable(tEnv, 'a, 'b, 'd, 'c, 'e)
-
- val unionDs = ds1.unionAll(ds2.select('a, 'b, 'c)).filter('b < 2).select('c)
-
- val results = unionDs.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList("Hi", "Hallo")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UserDefinedFunctionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UserDefinedFunctionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UserDefinedFunctionITCase.scala
deleted file mode 100644
index f438588..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/table/UserDefinedFunctionITCase.scala
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.runtime.datastream.table
-
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
-import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
-import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.scala.stream.utils.StreamTestData
-import org.apache.flink.table.expressions.utils.{Func13, Func18, RichFunc2}
-import org.apache.flink.table.runtime.datastream.StreamITCase
-import org.apache.flink.table.utils._
-import org.apache.flink.types.Row
-import org.junit.Assert._
-import org.junit.{Before, Test}
-
-import scala.collection.mutable
-
-class UserDefinedFunctionITCase extends StreamingMultipleProgramsTestBase {
-
- val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
- val tEnv: StreamTableEnvironment = TableEnvironment.getTableEnvironment(env)
-
- @Before
- def clear(): Unit = {
- StreamITCase.clear
- }
-
- @Test
- def testCrossJoin(): Unit = {
- val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
- val func0 = new TableFunc0
- val pojoFunc0 = new PojoTableFunc()
-
- val result = t
- .join(func0('c) as('d, 'e))
- .select('c, 'd, 'e)
- .join(pojoFunc0('c))
- .where('age > 20)
- .select('c, 'name, 'age)
- .toAppendStream[Row]
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList("Jack#22,Jack,22", "Anna#44,Anna,44")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testLeftOuterJoin(): Unit = {
- val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
- val func0 = new TableFunc0
-
- val result = t
- .leftOuterJoin(func0('c) as('d, 'e))
- .select('c, 'd, 'e)
- .toAppendStream[Row]
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "nosharp,null,null", "Jack#22,Jack,22",
- "John#19,John,19", "Anna#44,Anna,44")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testUserDefinedTableFunctionWithScalarFunction(): Unit = {
- val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
- val func0 = new TableFunc0
-
- val result = t
- .join(func0('c) as('d, 'e))
- .where(Func18('d, "J"))
- .select('c, 'd, 'e)
- .toAppendStream[Row]
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList("Jack#22,Jack,22", "John#19,John,19")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testUserDefinedTableFunctionWithParameter(): Unit = {
- val tableFunc1 = new RichTableFunc1
- tEnv.registerFunction("RichTableFunc1", tableFunc1)
- UserDefinedFunctionTestUtils.setJobParameters(env, Map("word_separator" -> " "))
- StreamITCase.testResults = mutable.MutableList()
-
- val result = StreamTestData.getSmall3TupleDataStream(env)
- .toTable(tEnv, 'a, 'b, 'c)
- .join(tableFunc1('c) as 's)
- .select('a, 's)
-
- val results = result.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList("3,Hello", "3,world")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testUserDefinedTableFunctionWithUserDefinedScalarFunction(): Unit = {
- val tableFunc1 = new RichTableFunc1
- val richFunc2 = new RichFunc2
- tEnv.registerFunction("RichTableFunc1", tableFunc1)
- tEnv.registerFunction("RichFunc2", richFunc2)
- UserDefinedFunctionTestUtils.setJobParameters(
- env,
- Map("word_separator" -> "#", "string.value" -> "test"))
- StreamITCase.testResults = mutable.MutableList()
-
- val result = StreamTestData.getSmall3TupleDataStream(env)
- .toTable(tEnv, 'a, 'b, 'c)
- .join(tableFunc1(richFunc2('c)) as 's)
- .select('a, 's)
-
- val results = result.toAppendStream[Row]
- results.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "1,Hi",
- "1,test",
- "2,Hello",
- "2,test",
- "3,Hello world",
- "3,test")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testTableFunctionConstructorWithParams(): Unit = {
- val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
- val config = Map("key1" -> "value1", "key2" -> "value2")
- val func30 = new TableFunc3(null)
- val func31 = new TableFunc3("OneConf_")
- val func32 = new TableFunc3("TwoConf_", config)
-
- val result = t
- .join(func30('c) as('d, 'e))
- .select('c, 'd, 'e)
- .join(func31('c) as ('f, 'g))
- .select('c, 'd, 'e, 'f, 'g)
- .join(func32('c) as ('h, 'i))
- .select('c, 'd, 'f, 'h, 'e, 'g, 'i)
- .toAppendStream[Row]
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "Anna#44,Anna,OneConf_Anna,TwoConf__key=key1_value=value1_Anna,44,44,44",
- "Anna#44,Anna,OneConf_Anna,TwoConf__key=key2_value=value2_Anna,44,44,44",
- "Jack#22,Jack,OneConf_Jack,TwoConf__key=key1_value=value1_Jack,22,22,22",
- "Jack#22,Jack,OneConf_Jack,TwoConf__key=key2_value=value2_Jack,22,22,22",
- "John#19,John,OneConf_John,TwoConf__key=key1_value=value1_John,19,19,19",
- "John#19,John,OneConf_John,TwoConf__key=key2_value=value2_John,19,19,19"
- )
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testScalarFunctionConstructorWithParams(): Unit = {
- val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
- val func0 = new Func13("default")
- val func1 = new Func13("Sunny")
- val func2 = new Func13("kevin2")
-
- val result = t.select(func0('c), func1('c),func2('c))
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "default-Anna#44,Sunny-Anna#44,kevin2-Anna#44",
- "default-Jack#22,Sunny-Jack#22,kevin2-Jack#22",
- "default-John#19,Sunny-John#19,kevin2-John#19",
- "default-nosharp,Sunny-nosharp,kevin2-nosharp"
- )
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- @Test
- def testTableFunctionWithVariableArguments(): Unit = {
- val varArgsFunc0 = new VarArgsFunc0
- tEnv.registerFunction("VarArgsFunc0", varArgsFunc0)
-
- val result = testData(env)
- .toTable(tEnv, 'a, 'b, 'c)
- .select('c)
- .join(varArgsFunc0("1", "2", 'c))
-
- result.addSink(new StreamITCase.StringSink[Row])
- env.execute()
-
- val expected = mutable.MutableList(
- "Anna#44,1",
- "Anna#44,2",
- "Anna#44,Anna#44",
- "Jack#22,1",
- "Jack#22,2",
- "Jack#22,Jack#22",
- "John#19,1",
- "John#19,2",
- "John#19,John#19",
- "nosharp,1",
- "nosharp,2",
- "nosharp,nosharp")
- assertEquals(expected.sorted, StreamITCase.testResults.sorted)
- }
-
- private def testData(
- env: StreamExecutionEnvironment)
- : DataStream[(Int, Long, String)] = {
-
- val data = new mutable.MutableList[(Int, Long, String)]
- data.+=((1, 1L, "Jack#22"))
- data.+=((2, 2L, "John#19"))
- data.+=((3, 2L, "Anna#44"))
- data.+=((4, 3L, "nosharp"))
- env.fromCollection(data)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
index 7ffc5f9..80ff55e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
@@ -36,13 +36,13 @@ import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
class HarnessTestBase {
- val longMinWithRetractAggFunction =
+ val longMinWithRetractAggFunction: String =
UserDefinedFunctionUtils.serialize(new LongMinWithRetractAggFunction)
- val longMaxWithRetractAggFunction =
+ val longMaxWithRetractAggFunction: String =
UserDefinedFunctionUtils.serialize(new LongMaxWithRetractAggFunction)
- val intSumWithRetractAggFunction =
+ val intSumWithRetractAggFunction: String =
UserDefinedFunctionUtils.serialize(new IntSumWithRetractAggFunction)
protected val MinMaxRowType = new RowTypeInfo(Array[TypeInformation[_]](
@@ -62,11 +62,11 @@ class HarnessTestBase {
protected val minMaxCRowType = new CRowTypeInfo(MinMaxRowType)
protected val sumCRowType = new CRowTypeInfo(SumRowType)
- protected val minMaxAggregates =
+ protected val minMaxAggregates: Array[AggregateFunction[_, _]] =
Array(new LongMinWithRetractAggFunction,
new LongMaxWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
- protected val sumAggregates =
+ protected val sumAggregates: Array[AggregateFunction[_, _]] =
Array(new IntSumWithRetractAggFunction).asInstanceOf[Array[AggregateFunction[_, _]]]
protected val minMaxAggregationStateType: RowTypeInfo =
@@ -90,11 +90,11 @@ class HarnessTestBase {
|
| fmin = (org.apache.flink.table.functions.aggfunctions.LongMinWithRetractAggFunction)
| org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
- | .deserialize("${longMinWithRetractAggFunction}");
+ | .deserialize("$longMinWithRetractAggFunction");
|
| fmax = (org.apache.flink.table.functions.aggfunctions.LongMaxWithRetractAggFunction)
| org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
- | .deserialize("${longMaxWithRetractAggFunction}");
+ | .deserialize("$longMaxWithRetractAggFunction");
| }
|
| public void setAggregationResults(
@@ -207,7 +207,7 @@ class HarnessTestBase {
|
|sum = (org.apache.flink.table.functions.aggfunctions.IntSumWithRetractAggFunction)
|org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
- |.deserialize("${intSumWithRetractAggFunction}");
+ |.deserialize("$intSumWithRetractAggFunction");
|}
|
| public final void setAggregationResults(
@@ -351,5 +351,4 @@ object HarnessTestBase {
value.row.getField(selectorField).asInstanceOf[T]
}
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala
new file mode 100644
index 0000000..0451534
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/SortProcessFunctionHarnessTest.scala
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.harness
+
+import java.lang.{Integer => JInt, Long => JLong}
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.java.typeutils.runtime.RowComparator
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.table.runtime.aggregate.{CollectionRowComparator, ProcTimeSortProcessFunction, RowTimeSortProcessFunction}
+import org.apache.flink.table.runtime.harness.SortProcessFunctionHarnessTest.TupleRowSelector
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.types.Row
+import org.junit.Test
+
+class SortProcessFunctionHarnessTest {
+
+ @Test
+ def testSortProcTimeHarnessPartitioned(): Unit = {
+
+ val rT = new RowTypeInfo(Array[TypeInformation[_]](
+ INT_TYPE_INFO,
+ LONG_TYPE_INFO,
+ INT_TYPE_INFO,
+ STRING_TYPE_INFO,
+ LONG_TYPE_INFO),
+ Array("a", "b", "c", "d", "e"))
+
+ val indexes = Array(1, 2)
+
+ val fieldComps = Array[TypeComparator[AnyRef]](
+ LONG_TYPE_INFO.createComparator(true, null).asInstanceOf[TypeComparator[AnyRef]],
+ INT_TYPE_INFO.createComparator(false, null).asInstanceOf[TypeComparator[AnyRef]] )
+ val booleanOrders = Array(true, false)
+
+
+ val rowComp = new RowComparator(
+ rT.getTotalFields,
+ indexes,
+ fieldComps,
+ new Array[TypeSerializer[AnyRef]](0), //used only for serialized comparisons
+ booleanOrders)
+
+ val collectionRowComparator = new CollectionRowComparator(rowComp)
+
+ val inputCRowType = CRowTypeInfo(rT)
+
+ val processFunction = new KeyedProcessOperator[Integer,CRow,CRow](
+ new ProcTimeSortProcessFunction(
+ inputCRowType,
+ collectionRowComparator))
+
+ val testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer,CRow,CRow](
+ processFunction,
+ new TupleRowSelector(0),
+ BasicTypeInfo.INT_TYPE_INFO)
+
+ testHarness.open()
+
+ testHarness.setProcessingTime(3)
+
+ // timestamp is ignored in processing time
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 11L: JLong), true), 1001))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 1: JInt, "aaa", 11L: JLong), true), 2002))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 2: JInt, "aaa", 11L: JLong), true), 2003))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2004))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2006))
+
+ //move the timestamp to ensure the execution
+ testHarness.setProcessingTime(1005)
+
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 1L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2007))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 3L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2007))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 2L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2007))
+
+ testHarness.setProcessingTime(1008)
+
+ val result = testHarness.getOutput
+
+ val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+ // all elements at the same proc timestamp have the same value
+ // elements should be sorted ascending on field 1 and descending on field 2
+ // (10,0) (11,1) (12,2) (12,1) (12,0)
+ // (1,0) (2,0)
+
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong),true), 4))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 11L: JLong),true), 4))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 2: JInt, "aaa", 11L: JLong),true), 4))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 1: JInt, "aaa", 11L: JLong),true), 4))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 0: JInt, "aaa", 11L: JLong),true), 4))
+
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 1L: JLong, 0: JInt, "aaa", 11L: JLong),true), 1006))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 2L: JLong, 0: JInt, "aaa", 11L: JLong),true), 1006))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 3L: JLong, 0: JInt, "aaa", 11L: JLong),true), 1006))
+
+ TestHarnessUtil.assertOutputEquals("Output was not correctly sorted.", expectedOutput, result)
+
+ testHarness.close()
+ }
+
+ @Test
+ def testSortRowTimeHarnessPartitioned(): Unit = {
+
+ val rT = new RowTypeInfo(Array[TypeInformation[_]](
+ INT_TYPE_INFO,
+ LONG_TYPE_INFO,
+ INT_TYPE_INFO,
+ STRING_TYPE_INFO,
+ LONG_TYPE_INFO),
+ Array("a", "b", "c", "d", "e"))
+
+ val indexes = Array(1, 2)
+
+ val fieldComps = Array[TypeComparator[AnyRef]](
+ LONG_TYPE_INFO.createComparator(true, null).asInstanceOf[TypeComparator[AnyRef]],
+ INT_TYPE_INFO.createComparator(false, null).asInstanceOf[TypeComparator[AnyRef]] )
+ val booleanOrders = Array(true, false)
+
+ val rowComp = new RowComparator(
+ rT.getTotalFields,
+ indexes,
+ fieldComps,
+ new Array[TypeSerializer[AnyRef]](0), //used only for serialized comparisons
+ booleanOrders)
+
+ val collectionRowComparator = new CollectionRowComparator(rowComp)
+
+ val inputCRowType = CRowTypeInfo(rT)
+
+ val processFunction = new KeyedProcessOperator[Integer,CRow,CRow](
+ new RowTimeSortProcessFunction(
+ inputCRowType,
+ Some(collectionRowComparator)))
+
+ val testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, CRow, CRow](
+ processFunction,
+ new TupleRowSelector(0),
+ BasicTypeInfo.INT_TYPE_INFO)
+
+ testHarness.open()
+
+ testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime)
+ testHarness.processWatermark(3)
+
+ // timestamp is ignored in processing time
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 11L: JLong), true), 1001))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 1: JInt, "aaa", 11L: JLong), true), 2002))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 13L: JLong, 2: JInt, "aaa", 11L: JLong), true), 2002))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 3: JInt, "aaa", 11L: JLong), true), 2002))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 14L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2002))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 3: JInt, "aaa", 11L: JLong), true), 2004))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2006))
+
+ // move watermark forward
+ testHarness.processWatermark(2007)
+
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 20L: JLong, 1: JInt, "aaa", 11L: JLong), true), 2008))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 14L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2002)) // too late
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 3: JInt, "aaa", 11L: JLong), true), 2019)) // too early
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 20L: JLong, 2: JInt, "aaa", 11L: JLong), true), 2008))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2010))
+ testHarness.processElement(new StreamRecord(new CRow(
+ Row.of(1: JInt, 19L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2008))
+
+ // move watermark forward
+ testHarness.processWatermark(2012)
+
+ val result = testHarness.getOutput
+
+ val expectedOutput = new ConcurrentLinkedQueue[Object]()
+
+ // all elements at the same proc timestamp have the same value
+ // elements should be sorted ascending on field 1 and descending on field 2
+ // (10,0) (11,1) (12,2) (12,1) (12,0)
+ expectedOutput.add(new Watermark(3))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 11L: JLong, 1: JInt, "aaa", 11L: JLong),true), 1001))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 3: JInt, "aaa", 11L: JLong),true), 2002))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 1: JInt, "aaa", 11L: JLong),true), 2002))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 13L: JLong, 2: JInt, "aaa", 11L: JLong),true), 2002))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 14L: JLong, 0: JInt, "aaa", 11L: JLong),true), 2002))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 12L: JLong, 3: JInt, "aaa", 11L: JLong),true), 2004))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong),true), 2006))
+ expectedOutput.add(new Watermark(2007))
+
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 19L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2008))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 20L: JLong, 2: JInt, "aaa", 11L: JLong), true), 2008))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 20L: JLong, 1: JInt, "aaa", 11L: JLong), true), 2008))
+ expectedOutput.add(new StreamRecord(new CRow(
+ Row.of(1: JInt, 10L: JLong, 0: JInt, "aaa", 11L: JLong), true), 2010))
+
+ expectedOutput.add(new Watermark(2012))
+
+ TestHarnessUtil.assertOutputEquals("Output was not correctly sorted.", expectedOutput, result)
+
+ testHarness.close()
+
+ }
+}
+
+object SortProcessFunctionHarnessTest {
+
+ /**
+ * Simple test class that returns a specified field as the selector function
+ */
+ class TupleRowSelector(private val selectorField: Int) extends KeySelector[CRow, Integer] {
+
+ override def getKey(value: CRow): Integer = {
+ value.row.getField(selectorField).asInstanceOf[Integer]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f1fafc0e/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
new file mode 100644
index 0000000..4c478de
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/TimeAttributesITCase.scala
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.stream
+
+import java.math.BigDecimal
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.apache.flink.table.runtime.utils.JavaPojos.Pojo1
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.plan.TimeIndicatorConversionTest.TableFunc
+import org.apache.flink.table.api.{TableEnvironment, Types}
+import org.apache.flink.table.expressions.{ExpressionParser, TimeIntervalUnit}
+import org.apache.flink.table.runtime.stream.TimeAttributesITCase.{TestPojo, TimestampWithEqualWatermark, TimestampWithEqualWatermarkPojo}
+import org.apache.flink.table.runtime.utils.StreamITCase
+import org.apache.flink.types.Row
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+/**
+ * Tests for access and materialization of time attributes.
+ */
+class TimeAttributesITCase extends StreamingMultipleProgramsTestBase {
+
+ val data = List(
+ (1L, 1, 1d, 1f, new BigDecimal("1"), "Hi"),
+ (2L, 2, 2d, 2f, new BigDecimal("2"), "Hallo"),
+ (3L, 2, 2d, 2f, new BigDecimal("2"), "Hello"),
+ (4L, 5, 5d, 5f, new BigDecimal("5"), "Hello"),
+ (7L, 3, 3d, 3f, new BigDecimal("3"), "Hello"),
+ (8L, 3, 3d, 3f, new BigDecimal("3"), "Hello world"),
+ (16L, 4, 4d, 4f, new BigDecimal("4"), "Hello world"))
+
+ @Test
+ def testCalcMaterialization(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val t = table.select('rowtime.cast(Types.STRING))
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.001",
+ "1970-01-01 00:00:00.002",
+ "1970-01-01 00:00:00.003",
+ "1970-01-01 00:00:00.004",
+ "1970-01-01 00:00:00.007",
+ "1970-01-01 00:00:00.008",
+ "1970-01-01 00:00:00.016")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testCalcMaterialization2(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val t = table
+ .filter('rowtime.cast(Types.LONG) > 4)
+ .select('rowtime, 'rowtime.floor(TimeIntervalUnit.DAY), 'rowtime.ceil(TimeIntervalUnit.DAY))
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.007,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
+ "1970-01-01 00:00:00.008,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0",
+ "1970-01-01 00:00:00.016,1970-01-01 00:00:00.0,1970-01-02 00:00:00.0")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testTableFunction(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(
+ tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string, 'proctime.proctime)
+ val func = new TableFunc
+
+ val t = table.join(func('rowtime, 'proctime, 'string) as 's).select('rowtime, 's)
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.001,1trueHi",
+ "1970-01-01 00:00:00.002,2trueHallo",
+ "1970-01-01 00:00:00.003,3trueHello",
+ "1970-01-01 00:00:00.004,4trueHello",
+ "1970-01-01 00:00:00.007,7trueHello",
+ "1970-01-01 00:00:00.008,8trueHello world",
+ "1970-01-01 00:00:00.016,16trueHello world")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testWindowAfterTableFunction(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(
+ tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string, 'proctime.proctime)
+ val func = new TableFunc
+
+ val t = table
+ .join(func('rowtime, 'proctime, 'string) as 's)
+ .window(Tumble over 5.millis on 'rowtime as 'w)
+ .groupBy('w)
+ .select('w.rowtime, 's.count)
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.004,4",
+ "1970-01-01 00:00:00.009,2",
+ "1970-01-01 00:00:00.019,1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testUnion(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(
+ tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val t = table.unionAll(table).select('rowtime)
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.001",
+ "1970-01-01 00:00:00.001",
+ "1970-01-01 00:00:00.002",
+ "1970-01-01 00:00:00.002",
+ "1970-01-01 00:00:00.003",
+ "1970-01-01 00:00:00.003",
+ "1970-01-01 00:00:00.004",
+ "1970-01-01 00:00:00.004",
+ "1970-01-01 00:00:00.007",
+ "1970-01-01 00:00:00.007",
+ "1970-01-01 00:00:00.008",
+ "1970-01-01 00:00:00.008",
+ "1970-01-01 00:00:00.016",
+ "1970-01-01 00:00:00.016")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testWindowWithAggregationOnRowtimeSql(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+ tEnv.registerTable("MyTable", table)
+
+ val t = tEnv.sql("SELECT COUNT(`rowtime`) FROM MyTable " +
+ "GROUP BY TUMBLE(rowtime, INTERVAL '0.003' SECOND)")
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1",
+ "2",
+ "2",
+ "2"
+ )
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testMultiWindow(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+
+ val t = table
+ .window(Tumble over 2.millis on 'rowtime as 'w)
+ .groupBy('w)
+ .select('w.rowtime as 'rowtime, 'int.count as 'int)
+ .window(Tumble over 4.millis on 'rowtime as 'w2)
+ .groupBy('w2)
+ .select('w2.rowtime, 'w2.end, 'int.count)
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.003,1970-01-01 00:00:00.004,2",
+ "1970-01-01 00:00:00.007,1970-01-01 00:00:00.008,2",
+ "1970-01-01 00:00:00.011,1970-01-01 00:00:00.012,1",
+ "1970-01-01 00:00:00.019,1970-01-01 00:00:00.02,1"
+ )
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testCalcMaterializationWithPojoType(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'rowtime.rowtime, 'int, 'double, 'float, 'bigdec, 'string)
+ tEnv.registerTable("T1", table)
+ val querySql = "select rowtime as ts, string as msg from T1"
+
+ val results = tEnv.sql(querySql).toAppendStream[Pojo1]
+ results.addSink(new StreamITCase.StringSink[Pojo1])
+ env.execute()
+
+ val expected = Seq(
+ "Pojo1{ts=1970-01-01 00:00:00.001, msg='Hi'}",
+ "Pojo1{ts=1970-01-01 00:00:00.002, msg='Hallo'}",
+ "Pojo1{ts=1970-01-01 00:00:00.003, msg='Hello'}",
+ "Pojo1{ts=1970-01-01 00:00:00.004, msg='Hello'}",
+ "Pojo1{ts=1970-01-01 00:00:00.007, msg='Hello'}",
+ "Pojo1{ts=1970-01-01 00:00:00.008, msg='Hello world'}",
+ "Pojo1{ts=1970-01-01 00:00:00.016, msg='Hello world'}")
+
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testPojoSupport(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val p1 = new TestPojo
+ p1.a = 12
+ p1.b = 42L
+ p1.c = "Test me."
+
+ val p2 = new TestPojo
+ p2.a = 13
+ p2.b = 43L
+ p2.c = "And me."
+
+ val stream = env
+ .fromElements(p1, p2)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermarkPojo)
+ // use aliases, swap all attributes, and skip b2
+ val table = stream.toTable(tEnv, ('b as 'b).rowtime, 'c as 'c, 'a as 'a)
+ // no aliases, no swapping
+ val table2 = stream.toTable(tEnv, 'a, 'b.rowtime, 'c)
+ // use proctime, no skipping
+ val table3 = stream.toTable(tEnv, 'a, 'b.rowtime, 'c, 'b2, 'proctime.proctime)
+
+ // Java expressions
+
+ // use aliases, swap all attributes, and skip b2
+ val table4 = stream.toTable(
+ tEnv,
+ ExpressionParser.parseExpressionList("(b as b).rowtime, c as c, a as a"): _*)
+ // no aliases, no swapping
+ val table5 = stream.toTable(
+ tEnv,
+ ExpressionParser.parseExpressionList("a, b.rowtime, c"): _*)
+
+ val t = table.select('b, 'c , 'a)
+ .unionAll(table2.select('b, 'c, 'a))
+ .unionAll(table3.select('b, 'c, 'a))
+ .unionAll(table4.select('b, 'c, 'a))
+ .unionAll(table5.select('b, 'c, 'a))
+
+ val results = t.toAppendStream[Row]
+ results.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ val expected = Seq(
+ "1970-01-01 00:00:00.042,Test me.,12",
+ "1970-01-01 00:00:00.042,Test me.,12",
+ "1970-01-01 00:00:00.042,Test me.,12",
+ "1970-01-01 00:00:00.042,Test me.,12",
+ "1970-01-01 00:00:00.042,Test me.,12",
+ "1970-01-01 00:00:00.043,And me.,13",
+ "1970-01-01 00:00:00.043,And me.,13",
+ "1970-01-01 00:00:00.043,And me.,13",
+ "1970-01-01 00:00:00.043,And me.,13",
+ "1970-01-01 00:00:00.043,And me.,13")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
+
+object TimeAttributesITCase {
+ class TimestampWithEqualWatermark
+ extends AssignerWithPunctuatedWatermarks[(Long, Int, Double, Float, BigDecimal, String)] {
+
+ override def checkAndGetNextWatermark(
+ lastElement: (Long, Int, Double, Float, BigDecimal, String),
+ extractedTimestamp: Long)
+ : Watermark = {
+ new Watermark(extractedTimestamp)
+ }
+
+ override def extractTimestamp(
+ element: (Long, Int, Double, Float, BigDecimal, String),
+ previousElementTimestamp: Long): Long = {
+ element._1
+ }
+ }
+
+ class TimestampWithEqualWatermarkPojo
+ extends AssignerWithPunctuatedWatermarks[TestPojo] {
+
+ override def checkAndGetNextWatermark(
+ lastElement: TestPojo,
+ extractedTimestamp: Long)
+ : Watermark = {
+ new Watermark(extractedTimestamp)
+ }
+
+ override def extractTimestamp(
+ element: TestPojo,
+ previousElementTimestamp: Long): Long = {
+ element.b
+ }
+ }
+
+ class TestPojo() {
+ var a: Int = _
+ var b: Long = _
+ var b2: String = "skip me"
+ var c: String = _
+ }
+}