You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2017/05/09 16:54:17 UTC
[1/4] flink git commit: [FLINK-6479] [table] Fix IOOBE in
DataStreamGroupWindowAggregate.
Repository: flink
Updated Branches:
refs/heads/release-1.3 fd910f2c9 -> 28a89d1ca
[FLINK-6479] [table] Fix IOOBE in DataStreamGroupWindowAggregate.
This closes #3841.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4556b49c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4556b49c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4556b49c
Branch: refs/heads/release-1.3
Commit: 4556b49c80108e83d67a56ac5039f631dd681a96
Parents: fd910f2
Author: sunjincheng121 <su...@gmail.com>
Authored: Mon May 8 16:41:31 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue May 9 18:52:20 2017 +0200
----------------------------------------------------------------------
.../DataStreamGroupWindowAggregate.scala | 2 +-
.../table/GroupWindowAggregationsITCase.scala | 32 +++++++++++++++
.../scala/stream/table/GroupWindowTest.scala | 41 +++++++++++++++++++-
3 files changed, 73 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4556b49c/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 1be1896..c38e5af 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -134,7 +134,7 @@ class DataStreamGroupWindowAggregate(
namedAggregates,
namedProperties)
- val keyedAggOpName = s"groupBy: (${groupingToString(schema.logicalType, grouping)}), " +
+ val keyedAggOpName = s"groupBy: (${groupingToString(inputSchema.logicalType, grouping)}), " +
s"window: ($window), " +
s"select: ($aggString)"
val nonKeyedAggOpName = s"window: ($window), select: ($aggString)"
http://git-wip-us.apache.org/repos/asf/flink/blob/4556b49c/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
index 2c027a9..846fe3e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowAggregationsITCase.scala
@@ -175,6 +175,38 @@ class GroupWindowAggregationsITCase extends StreamingMultipleProgramsTestBase {
"Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
+
+ @Test
+ def testGroupWindowWithoutKeyInProjection(): Unit = {
+ val data = List(
+ (1L, 1, "Hi", 1, 1),
+ (2L, 2, "Hello", 2, 2),
+ (4L, 2, "Hello", 2, 2),
+ (8L, 3, "Hello world", 3, 3),
+ (16L, 3, "Hello world", 3, 3))
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setParallelism(1)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string, 'int2, 'int3, 'proctime.proctime)
+
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+ .groupBy('w, 'int2, 'int3, 'string)
+ .select(weightAvgFun('long, 'int))
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq("12", "8", "2", "3", "1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
}
object GroupWindowAggregationsITCase {
http://git-wip-us.apache.org/repos/asf/flink/blob/4556b49c/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
index 0573ff3..ef071b7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/table/GroupWindowTest.scala
@@ -19,7 +19,7 @@
package org.apache.flink.table.api.scala.stream.table
import org.apache.flink.api.scala._
-import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.WeightedAvgWithMerge
+import org.apache.flink.table.api.java.utils.UserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
import org.apache.flink.table.api.ValidationException
import org.apache.flink.table.api.scala._
import org.apache.flink.table.expressions.WindowReference
@@ -792,6 +792,45 @@ class GroupWindowTest extends TableTestBase {
}
@Test
+ def testSlidingWindowWithUDAF(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String, Int, Int)](
+ 'long,
+ 'int,
+ 'string,
+ 'int2,
+ 'int3,
+ 'proctime.proctime)
+
+ val weightAvgFun = new WeightedAvg
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows on 'proctime as 'w)
+ .groupBy('w, 'int2, 'int3, 'string)
+ .select(weightAvgFun('long, 'int))
+
+ val expected =
+ unaryNode(
+ "DataStreamCalc",
+ unaryNode(
+ "DataStreamGroupWindowAggregate",
+ streamTableNode(0),
+ term("groupBy", "string, int2, int3"),
+ term("window", SlidingGroupWindow(WindowReference("w"), 'proctime, 2.rows, 1.rows)),
+ term(
+ "select",
+ "string",
+ "int2",
+ "int3",
+ "WeightedAvg(long, int) AS TMP_0")
+ ),
+ term("select","TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
def testSlideWindowStartEnd(): Unit = {
val util = streamTestUtil()
val table = util.addTable[(Long, Int, String)]('long, 'int, 'string, 'rowtime.rowtime)
[3/4] flink git commit: [FLINK-6486] [table] Pass RowTypeInfo to
CodeGenerator instead of CRowTypeInfo.
Posted by fh...@apache.org.
[FLINK-6486] [table] Pass RowTypeInfo to CodeGenerator instead of CRowTypeInfo.
This closes #3850.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/dd799c74
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/dd799c74
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/dd799c74
Branch: refs/heads/release-1.3
Commit: dd799c746cc464550222b9b126e3f60c5259df9f
Parents: 34cff35
Author: Hequn Cheng <ch...@gmail.com>
Authored: Mon May 8 20:55:51 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue May 9 18:53:14 2017 +0200
----------------------------------------------------------------------
.../table/plan/nodes/datastream/DataStreamGroupAggregate.scala | 2 +-
.../plan/nodes/datastream/DataStreamGroupWindowAggregate.scala | 2 +-
.../table/plan/nodes/datastream/DataStreamOverAggregate.scala | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/dd799c74/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 18f1fc8..506c0cb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -115,7 +115,7 @@ class DataStreamGroupAggregate(
val generator = new CodeGenerator(
tableEnv.getConfig,
false,
- inputDS.getType)
+ inputSchema.physicalTypeInfo)
val aggString = aggregationToString(
inputSchema.logicalType,
http://git-wip-us.apache.org/repos/asf/flink/blob/dd799c74/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index c38e5af..ef207b0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -142,7 +142,7 @@ class DataStreamGroupWindowAggregate(
val generator = new CodeGenerator(
tableEnv.getConfig,
false,
- inputDS.getType)
+ inputSchema.physicalTypeInfo)
val needMerge = window match {
case SessionGroupWindow(_, _, _) => true
http://git-wip-us.apache.org/repos/asf/flink/blob/dd799c74/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index e823cd6..4061242 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -116,7 +116,7 @@ class DataStreamOverAggregate(
val generator = new CodeGenerator(
tableEnv.getConfig,
false,
- inputDS.getType)
+ inputSchema.physicalTypeInfo)
val timeType = schema.logicalType
.getFieldList
[2/4] flink git commit: [FLINK-6436] [table] Fix code-gen bug when
using a scalar UDF in a UDTF join condition.
Posted by fh...@apache.org.
[FLINK-6436] [table] Fix code-gen bug when using a scalar UDF in a UDTF join condition.
This closes #3815.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/34cff356
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/34cff356
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/34cff356
Branch: refs/heads/release-1.3
Commit: 34cff3563357bbcac8d516391e03feb37c89641b
Parents: 4556b49
Author: godfreyhe <go...@163.com>
Authored: Wed May 3 20:59:34 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue May 9 18:52:45 2017 +0200
----------------------------------------------------------------------
.../table/plan/nodes/CommonCorrelate.scala | 13 ++++++++++-
.../utils/UserDefinedScalarFunctions.scala | 9 ++++++--
.../DataSetUserDefinedFunctionITCase.scala | 24 +++++++++++++++++---
.../DataStreamUserDefinedFunctionITCase.scala | 22 ++++++++++++++++--
4 files changed, 60 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/34cff356/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 44a109e3..c95f2f7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.plan.nodes
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rex.{RexCall, RexNode}
+import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle}
import org.apache.calcite.sql.SemiJoinType
import org.apache.flink.api.common.functions.FlatMapFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -143,6 +143,17 @@ trait CommonCorrelate[T] {
|getCollector().collect(${crossResultExpr.resultTerm});
|""".stripMargin
} else {
+
+ // adjust indicies of InputRefs to adhere to schema expected by generator
+ val changeInputRefIndexShuttle = new RexShuttle {
+ override def visitInputRef(inputRef: RexInputRef): RexNode = {
+ new RexInputRef(inputSchema.physicalArity + inputRef.getIndex, inputRef.getType)
+ }
+ }
+ // Run generateExpression to add init statements (ScalarFunctions) of condition to generator.
+ // The generated expression is discarded.
+ generator.generateExpression(condition.get.accept(changeInputRefIndexShuttle))
+
val filterGenerator = new CodeGenerator(config, false, udtfTypeInfo, None, pojoFieldMapping)
filterGenerator.input1Term = filterGenerator.input2Term
val filterCondition = filterGenerator.generateExpression(condition.get)
http://git-wip-us.apache.org/repos/asf/flink/blob/34cff356/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
index 8972a77..5285569 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/utils/UserDefinedScalarFunctions.scala
@@ -25,11 +25,10 @@ import org.apache.flink.table.api.Types
import org.apache.flink.table.functions.{ScalarFunction, FunctionContext}
import org.junit.Assert
+import scala.annotation.varargs
import scala.collection.mutable
import scala.io.Source
-import scala.annotation.varargs
-
case class SimplePojo(name: String, age: Int)
object Func0 extends ScalarFunction {
@@ -263,3 +262,9 @@ object Func17 extends ScalarFunction {
a.mkString(", ")
}
}
+
+object Func18 extends ScalarFunction {
+ def eval(str: String, prefix: String): Boolean = {
+ str.startsWith(prefix)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/34cff356/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
index 20bbf8b..b69dd49 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/dataset/DataSetUserDefinedFunctionITCase.scala
@@ -24,9 +24,9 @@ import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.table.api.TableEnvironment
import org.apache.flink.table.api.java.utils.UserDefinedTableFunctions.JavaTableFunc0
import org.apache.flink.table.api.scala._
-import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
import org.apache.flink.table.api.scala.batch.utils.TableProgramsClusterTestBase
-import org.apache.flink.table.expressions.utils.{Func13, RichFunc2}
+import org.apache.flink.table.api.scala.batch.utils.TableProgramsTestBase.TableConfigMode
+import org.apache.flink.table.expressions.utils.{Func1, Func13, Func18, RichFunc2}
import org.apache.flink.table.utils._
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.TestBaseUtils
@@ -143,7 +143,7 @@ class DataSetUserDefinedFunctionITCase(
val pojo = new PojoTableFunc()
val result = in
.join(pojo('c))
- .where(('age > 20))
+ .where('age > 20)
.select('c, 'name, 'age)
.toDataSet[Row]
@@ -171,6 +171,24 @@ class DataSetUserDefinedFunctionITCase(
}
@Test
+ def testUserDefinedTableFunctionWithScalarFunctionInCondition(): Unit = {
+ val env = ExecutionEnvironment.getExecutionEnvironment
+ val tableEnv = TableEnvironment.getTableEnvironment(env, config)
+ val in = testData(env).toTable(tableEnv).as('a, 'b, 'c)
+ val func0 = new TableFunc0
+
+ val result = in
+ .join(func0('c))
+ .where(Func18('name, "J") && (Func1('a) < 3) && Func1('age) > 20)
+ .select('c, 'name, 'age)
+ .toDataSet[Row]
+
+ val results = result.collect()
+ val expected = "Jack#22,Jack,22"
+ TestBaseUtils.compareResultAsText(results.asJava, expected)
+ }
+
+ @Test
def testLongAndTemporalTypes(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tableEnv = TableEnvironment.getTableEnvironment(env, config)
http://git-wip-us.apache.org/repos/asf/flink/blob/34cff356/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
index 2e8a065..b3d9c6f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/datastream/DataStreamUserDefinedFunctionITCase.scala
@@ -23,7 +23,7 @@ import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
import org.apache.flink.table.api.TableEnvironment
import org.apache.flink.table.api.scala._
import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData}
-import org.apache.flink.table.expressions.utils.{Func13, RichFunc2}
+import org.apache.flink.table.expressions.utils.{Func13, Func18, RichFunc2}
import org.apache.flink.table.utils._
import org.apache.flink.types.Row
import org.junit.Assert._
@@ -51,7 +51,7 @@ class DataStreamUserDefinedFunctionITCase extends StreamingMultipleProgramsTestB
.join(func0('c) as('d, 'e))
.select('c, 'd, 'e)
.join(pojoFunc0('c))
- .where(('age > 20))
+ .where('age > 20)
.select('c, 'name, 'age)
.toDataStream[Row]
@@ -82,6 +82,24 @@ class DataStreamUserDefinedFunctionITCase extends StreamingMultipleProgramsTestB
}
@Test
+ def testUserDefinedTableFunctionWithScalarFunction(): Unit = {
+ val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
+ val func0 = new TableFunc0
+
+ val result = t
+ .join(func0('c) as('d, 'e))
+ .where(Func18('d, "J"))
+ .select('c, 'd, 'e)
+ .toDataStream[Row]
+
+ result.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = mutable.MutableList("Jack#22,Jack,22", "John#19,John,19")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
def testUserDefinedTableFunctionWithParameter(): Unit = {
val tableFunc1 = new RichTableFunc1
tEnv.registerFunction("RichTableFunc1", tableFunc1)
[4/4] flink git commit: [FLINK-6476] [table] Add support to convert
DataSet[Row] and DataStream[Row] to Table.
Posted by fh...@apache.org.
[FLINK-6476] [table] Add support to convert DataSet[Row] and DataStream[Row] to Table.
This closes #3849.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/28a89d1c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/28a89d1c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/28a89d1c
Branch: refs/heads/release-1.3
Commit: 28a89d1cac79063245bbc1ad9d262e3bc94b17b9
Parents: dd799c7
Author: rtudoran <tu...@ymail.com>
Authored: Mon May 8 20:30:57 2017 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue May 9 18:53:33 2017 +0200
----------------------------------------------------------------------
.../flink/table/api/TableEnvironment.scala | 18 +++++++++
.../table/api/java/stream/sql/SqlITCase.java | 42 ++++++++++++++++++++
.../flink/table/TableEnvironmentTest.scala | 22 ++++++++++
.../table/api/scala/stream/sql/SqlITCase.scala | 36 +++++++++++++++++
4 files changed, 118 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/28a89d1c/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index bb0de3e..bf4a8e0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -63,6 +63,7 @@ import org.apache.flink.table.sinks.TableSink
import org.apache.flink.table.sources.{DefinedFieldNames, TableSource}
import org.apache.flink.table.validate.FunctionCatalog
import org.apache.flink.types.Row
+import org.apache.flink.api.java.typeutils.RowTypeInfo
import _root_.scala.collection.JavaConverters._
import _root_.scala.collection.mutable.HashMap
@@ -677,6 +678,23 @@ abstract class TableEnvironment(val config: TableConfig) {
case _ => throw new TableException(
"Field reference expression or alias on field expression expected.")
}
+ case r: RowTypeInfo => {
+ exprs.zipWithIndex flatMap {
+ case (UnresolvedFieldReference(name), idx) =>
+ Some((idx, name))
+ case (Alias(UnresolvedFieldReference(origName), name, _), _) =>
+ val idx = r.getFieldIndex(origName)
+ if (idx < 0) {
+ throw new TableException(s"$origName is not a field of type $r")
+ }
+ Some((idx, name))
+ case (_: TimeAttribute, _) =>
+ None
+ case _ => throw new TableException(
+ "Field reference expression or alias on field expression expected.")
+ }
+
+ }
case tpe => throw new TableException(
s"Source of type $tpe cannot be converted into Table.")
}
http://git-wip-us.apache.org/repos/asf/flink/blob/28a89d1c/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
index 7c01d2b..0c0b37e 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/api/java/stream/sql/SqlITCase.java
@@ -30,11 +30,53 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase;
import org.apache.flink.table.api.java.stream.utils.StreamTestData;
import org.junit.Test;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
import java.util.ArrayList;
import java.util.List;
public class SqlITCase extends StreamingMultipleProgramsTestBase {
+
+ @Test
+ public void testRowRegisterRowWithNames() throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env);
+ StreamITCase.clear();
+
+ List<Row> data = new ArrayList<>();
+ data.add(Row.of(1, 1L, "Hi"));
+ data.add(Row.of(2, 2L, "Hello"));
+ data.add(Row.of(3, 2L, "Hello world"));
+
+ TypeInformation<?>[] types = {
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO};
+ String names[] = {"a","b","c"};
+
+ RowTypeInfo typeInfo = new RowTypeInfo(types, names);
+
+ DataStream<Row> ds = env.fromCollection(data).returns(typeInfo);
+
+ Table in = tableEnv.fromDataStream(ds, "a,b,c");
+ tableEnv.registerTable("MyTableRow", in);
+
+ String sqlQuery = "SELECT a,c FROM MyTableRow";
+ Table result = tableEnv.sql(sqlQuery);
+
+ DataStream<Row> resultSet = tableEnv.toDataStream(result, Row.class);
+ resultSet.addSink(new StreamITCase.StringSink());
+ env.execute();
+
+ List<String> expected = new ArrayList<>();
+ expected.add("1,Hi");
+ expected.add("2,Hello");
+ expected.add("3,Hello world");
+
+ StreamITCase.compareWithList(expected);
+ }
@Test
public void testSelect() throws Exception {
http://git-wip-us.apache.org/repos/asf/flink/blob/28a89d1c/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
index ba3b591..98a8edb 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/TableEnvironmentTest.scala
@@ -54,6 +54,28 @@ class TableEnvironmentTest extends TableTestBase {
val genericRowType = new GenericTypeInfo[Row](classOf[Row])
@Test
+ def testGetFieldInfoRow(): Unit = {
+ val fieldInfo = tEnv.getFieldInfo(rowType)
+
+ fieldInfo._1.zip(Array("f0", "f1", "f2")).foreach(x => assertEquals(x._2, x._1))
+ fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+ }
+
+ @Test
+ def testGetFieldInfoRowNames(): Unit = {
+ val fieldInfo = tEnv.getFieldInfo(
+ rowType,
+ Array(
+ UnresolvedFieldReference("name1"),
+ UnresolvedFieldReference("name2"),
+ UnresolvedFieldReference("name3")
+ ))
+
+ fieldInfo._1.zip(Array("name1", "name2", "name3")).foreach(x => assertEquals(x._2, x._1))
+ fieldInfo._2.zip(Array(0, 1, 2)).foreach(x => assertEquals(x._2, x._1))
+ }
+
+ @Test
def testGetFieldInfoTuple(): Unit = {
val fieldInfo = tEnv.getFieldInfo(tupleType)
http://git-wip-us.apache.org/repos/asf/flink/blob/28a89d1c/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
index 4147358..ba8c185 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/scala/stream/sql/SqlITCase.scala
@@ -23,12 +23,48 @@ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.table.api.TableEnvironment
import org.apache.flink.table.api.scala._
import org.apache.flink.table.api.scala.stream.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.typeutils.RowTypeInfo
+import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.types.Row
import org.junit.Assert._
import org.junit._
class SqlITCase extends StreamingWithStateTestBase {
+ /** test row stream registered table **/
+ @Test
+ def testRowRegister(): Unit = {
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+
+ val sqlQuery = "SELECT * FROM MyTableRow WHERE c < 3"
+
+ val data = List(
+ Row.of("Hello", "Worlds", Int.box(1)),
+ Row.of("Hello", "Hiden", Int.box(5)),
+ Row.of("Hello again", "Worlds", Int.box(2)))
+
+ implicit val tpe: TypeInformation[Row] = new RowTypeInfo(
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.INT_TYPE_INFO) // tpe is automatically
+
+ val ds = env.fromCollection(data)
+
+ val t = ds.toTable(tEnv).as('a, 'b, 'c)
+ tEnv.registerTable("MyTableRow", t)
+
+ val result = tEnv.sql(sqlQuery).toDataStream[Row]
+ result.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = List("Hello,Worlds,1","Hello again,Worlds,2")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
/** test unbounded groupby (without window) **/
@Test
def testUnboundedGroupby(): Unit = {