You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/10/26 21:11:23 UTC
[1/5] flink git commit: [FLINK-4691] [table] Add group-windows for
streaming tables to Table API.
Repository: flink
Updated Branches:
refs/heads/master baf057a48 -> de03e0cea
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
new file mode 100644
index 0000000..96fd787
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.table._
+import org.apache.flink.api.table._
+import org.apache.flink.api.table.expressions.{RowtimeAttribute, WindowReference}
+import org.apache.flink.api.table.plan.logical._
+import org.apache.flink.api.table.utils.TableTestBase
+import org.apache.flink.api.table.utils.TableTestUtil.{streamTableNode, term, unaryNode}
+import org.junit.{Ignore, Test}
+
+class GroupWindowTest extends TableTestBase {
+
+ // batch windows are not supported yet
+ @Test(expected = classOf[ValidationException])
+ def testInvalidBatchWindow(): Unit = {
+ val util = batchTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Session withGap 100.milli as 'string)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidWindowProperty(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .select('string, 'string.start) // property in non windowed table
+ }
+
+ @Test(expected = classOf[TableException])
+ def testInvalidRowtime1(): Unit = {
+ val util = streamTestUtil()
+ // rowtime attribute must not be a field name
+ util.addTable[(Long, Int, String)]('rowtime, 'long, 'int, 'string)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidRowtime2(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .select('string, 'int as 'rowtime) // rowtime attribute must not be an alias
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidRowtime3(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table.as('rowtime, 'myint, 'mystring) // rowtime attribute must not be an alias
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidRowtime4(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ // only rowtime is a valid time attribute in a stream environment
+ .window(Tumble over 50.milli on 'string)
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidTumblingSize(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Tumble over "WRONG") // string is not a valid interval
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidSlidingSize(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Slide over "WRONG" every "WRONG") // string is not a valid interval
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidSlidingSlide(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Slide over 12.rows every 1.minute) // row and time intervals may not be mixed
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidSessionGap(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Session withGap 10.rows) // row interval is not valid for session windows
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidWindowAlias1(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Session withGap 100.milli as 1 + 1) // expression instead of a symbol
+ .select('string, 'int.count)
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidWindowAlias2(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ table
+ .groupBy('string)
+ .window(Session withGap 100.milli as 'string) // field name "string" is already present
+ .select('string, 'int.count)
+ }
+
+ @Test
+ def testProcessingTimeTumblingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 50.milli)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", ProcessingTimeTumblingGroupWindow(None, 50.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testProcessingTimeTumblingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 2.rows)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", ProcessingTimeTumblingGroupWindow(None, 2.rows)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testEventTimeTumblingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 5.milli on 'rowtime)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", EventTimeTumblingGroupWindow(None, RowtimeAttribute(), 5.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ @Ignore // see comments in DataStreamAggregate
+ def testEventTimeTumblingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 2.rows on 'rowtime)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", EventTimeTumblingGroupWindow(None, RowtimeAttribute(), 2.rows)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testProcessingTimeSlidingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 50.milli every 50.milli)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", ProcessingTimeSlidingGroupWindow(None, 50.milli, 50.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 2.rows every 1.rows)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", ProcessingTimeSlidingGroupWindow(None, 2.rows, 1.rows)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testEventTimeSlidingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 8.milli every 10.milli on 'rowtime)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", EventTimeSlidingGroupWindow(None, RowtimeAttribute(), 8.milli, 10.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ @Ignore // see comments in DataStreamAggregate
+ def testEventTimeSlidingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 2.rows every 1.rows on 'rowtime)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", EventTimeSlidingGroupWindow(None, RowtimeAttribute(), 2.rows, 1.rows)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testEventTimeSessionGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Session withGap 7.milli on 'rowtime)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", EventTimeSessionGroupWindow(None, RowtimeAttribute(), 7.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllProcessingTimeTumblingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 50.milli)
+ .select('string, 'int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window", ProcessingTimeTumblingGroupWindow(None, 50.milli)),
+ term("select", "string", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Tumble over 2.rows)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", ProcessingTimeTumblingGroupWindow(None, 2.rows)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllEventTimeTumblingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Tumble over 5.milli on 'rowtime)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", EventTimeTumblingGroupWindow(None, RowtimeAttribute(), 5.milli)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ @Ignore // see comments in DataStreamAggregate
+ def testAllEventTimeTumblingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Tumble over 2.rows on 'rowtime)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", EventTimeTumblingGroupWindow(None, RowtimeAttribute(), 2.rows)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+
+ @Test
+ def testAllProcessingTimeSlidingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Slide over 50.milli every 50.milli)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", ProcessingTimeSlidingGroupWindow(None, 50.milli, 50.milli)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllProcessingTimeSlidingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", ProcessingTimeSlidingGroupWindow(None, 2.rows, 1.rows)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllEventTimeSlidingGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Slide over 8.milli every 10.milli on 'rowtime)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", EventTimeSlidingGroupWindow(None, RowtimeAttribute(), 8.milli, 10.milli)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ @Ignore // see comments in DataStreamAggregate
+ def testAllEventTimeSlidingGroupWindowOverCount(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Slide over 2.rows every 1.rows on 'rowtime)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", EventTimeSlidingGroupWindow(None, RowtimeAttribute(), 2.rows, 1.rows)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testAllEventTimeSessionGroupWindowOverTime(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Session withGap 7.milli on 'rowtime)
+ .select('int.count)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("window", EventTimeSessionGroupWindow(None, RowtimeAttribute(), 7.milli)),
+ term("select", "COUNT(int) AS TMP_0")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testTumbleWindowStartEnd(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 5.milli on 'rowtime as 'w)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window",
+ EventTimeTumblingGroupWindow(
+ Some(WindowReference("w")),
+ RowtimeAttribute(),
+ 5.milli)),
+ term("select",
+ "string",
+ "COUNT(int) AS TMP_0",
+ "start(WindowReference(w)) AS TMP_1",
+ "end(WindowReference(w)) AS TMP_2")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testSlideWindowStartEnd(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val expected = unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window",
+ EventTimeSlidingGroupWindow(
+ Some(WindowReference("w")),
+ RowtimeAttribute(),
+ 10.milli,
+ 5.milli)),
+ term("select",
+ "string",
+ "COUNT(int) AS TMP_0",
+ "start(WindowReference(w)) AS TMP_1",
+ "end(WindowReference(w)) AS TMP_2")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testSessionWindowStartWithTwoEnd(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Session withGap 3.milli on 'rowtime as 'w)
+ .select('w.end, 'string, 'int.count, 'w.start, 'w.end)
+
+ val expected = unaryNode(
+ "DataStreamCalc",
+ unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window",
+ EventTimeSessionGroupWindow(
+ Some(WindowReference("w")),
+ RowtimeAttribute(),
+ 3.milli)),
+ term("select",
+ "string",
+ "COUNT(int) AS TMP_1",
+ "end(WindowReference(w)) AS TMP_0",
+ "start(WindowReference(w)) AS TMP_2",
+ "end(WindowReference(w)) AS TMP_3")
+ ),
+ term("select", "TMP_0", "string", "TMP_1", "TMP_2", "TMP_3")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/UnsupportedOpsTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/UnsupportedOpsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/UnsupportedOpsTest.scala
index 8ce1472..6d1a62e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/UnsupportedOpsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/UnsupportedOpsTest.scala
@@ -35,14 +35,6 @@ class UnsupportedOpsTest extends StreamingMultipleProgramsTestBase {
}
@Test(expected = classOf[ValidationException])
- def testGroupBy(): Unit = {
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- val tEnv = TableEnvironment.getTableEnvironment(env)
- StreamTestData.getSmall3TupleDataStream(env).toTable(tEnv)
- .groupBy('_1)
- }
-
- @Test(expected = classOf[ValidationException])
def testDistinct(): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val tEnv = TableEnvironment.getTableEnvironment(env)
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/utils/TableTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/utils/TableTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/utils/TableTestBase.scala
index fd43ed4..ce693ff 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/utils/TableTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/table/utils/TableTestBase.scala
@@ -27,7 +27,7 @@ import org.apache.flink.api.table.expressions.Expression
import org.apache.flink.api.table.{Table, TableEnvironment}
import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
-import org.junit.Assert
+import org.junit.Assert.assertEquals
import org.mockito.Mockito.{mock, when}
/**
@@ -46,6 +46,13 @@ class TableTestBase {
}
abstract class TableTestUtil {
+
+ private var counter = 0
+
+ def addTable[T: TypeInformation](fields: Expression*): Table = {
+ addTable[T](s"Table${counter += 1}", fields: _*)
+ }
+
def addTable[T: TypeInformation](name: String, fields: Expression*): Table
def verifySql(query: String, expected: String): Unit
def verifyTable(resultTable: Table, expected: String): Unit
@@ -58,18 +65,18 @@ object TableTestUtil {
def unaryNode(node: String, input: String, term: String*): String = {
s"""$node(${term.mkString(", ")})
- | $input
+ |$input
|""".stripMargin
}
def binaryNode(node: String, left: String, right: String, term: String*): String = {
s"""$node(${term.mkString(", ")})
- | $left
- | $right
+ |$left
+ |$right
|""".stripMargin
}
- def term(term: String, value: String*): String = {
+ def term(term: AnyRef, value: AnyRef*): String = {
s"$term=[${value.mkString(", ")}]"
}
@@ -110,7 +117,9 @@ case class BatchTableTestUtil() extends TableTestUtil {
val relNode = resultTable.getRelNode
val optimized = tEnv.optimize(relNode)
val actual = RelOptUtil.toString(optimized)
- Assert.assertEquals(expected, actual)
+ assertEquals(
+ expected.split("\n").map(_.trim).mkString("\n"),
+ actual.split("\n").map(_.trim).mkString("\n"))
}
}
@@ -143,6 +152,8 @@ case class StreamTableTestUtil() extends TableTestUtil {
val relNode = resultTable.getRelNode
val optimized = tEnv.optimize(relNode)
val actual = RelOptUtil.toString(optimized)
- Assert.assertEquals(expected, actual)
+ assertEquals(
+ expected.split("\n").map(_.trim).mkString("\n"),
+ actual.split("\n").map(_.trim).mkString("\n"))
}
}
[4/5] flink git commit: [FLINK-4691] [table] Add group-windows for
streaming tables to Table API.
Posted by fh...@apache.org.
[FLINK-4691] [table] Add group-windows for streaming tables to Table API.
This closes #2562.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/44f3977e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/44f3977e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/44f3977e
Branch: refs/heads/master
Commit: 44f3977e7fee44a3601b651a523c4ab37ed5a15d
Parents: baf057a
Author: twalthr <tw...@apache.org>
Authored: Thu Aug 25 09:19:53 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Oct 26 23:03:37 2016 +0200
----------------------------------------------------------------------
docs/dev/table_api.md | 444 +++++++++++++-
.../flink/api/java/table/groupWindows.scala | 84 +++
.../flink/api/scala/table/expressionDsl.scala | 30 +-
.../flink/api/scala/table/groupWindows.scala | 85 +++
.../flink/api/table/BatchTableEnvironment.scala | 4 +-
.../flink/api/table/FlinkRelBuilder.scala | 34 +-
.../flink/api/table/FlinkTypeFactory.scala | 10 +-
.../api/table/StreamTableEnvironment.scala | 45 +-
.../flink/api/table/TableEnvironment.scala | 8 +-
.../org/apache/flink/api/table/Types.scala | 6 +-
.../flink/api/table/codegen/CodeGenUtils.scala | 10 +-
.../table/codegen/calls/ScalarOperators.scala | 28 +-
.../org/apache/flink/api/table/exceptions.scala | 13 +-
.../api/table/expressions/Expression.scala | 4 +-
.../table/expressions/ExpressionParser.scala | 36 +-
.../api/table/expressions/ExpressionUtils.scala | 21 +-
.../api/table/expressions/InputTypeSpec.scala | 2 +-
.../api/table/expressions/aggregations.scala | 9 +-
.../api/table/expressions/arithmetic.scala | 8 +-
.../flink/api/table/expressions/call.scala | 6 +-
.../flink/api/table/expressions/cast.scala | 2 +-
.../api/table/expressions/comparison.scala | 6 +-
.../api/table/expressions/fieldExpression.scala | 51 +-
.../flink/api/table/expressions/literals.scala | 17 +-
.../flink/api/table/expressions/logic.scala | 6 +-
.../api/table/expressions/mathExpressions.scala | 6 +-
.../flink/api/table/expressions/ordering.scala | 2 +-
.../table/expressions/stringExpressions.scala | 12 +-
.../flink/api/table/expressions/time.scala | 18 +-
.../table/expressions/windowProperties.scala | 57 ++
.../api/table/plan/ProjectionTranslator.scala | 105 ++++
.../api/table/plan/RexNodeTranslator.scala | 88 ---
.../api/table/plan/logical/LogicalNode.scala | 14 +-
.../api/table/plan/logical/LogicalWindow.scala | 36 ++
.../api/table/plan/logical/Resolvable.scala | 43 ++
.../api/table/plan/logical/groupWindows.scala | 258 ++++++++
.../api/table/plan/logical/operators.scala | 144 ++++-
.../logical/rel/LogicalWindowAggregate.scala | 114 ++++
.../api/table/plan/nodes/FlinkAggregate.scala | 68 ++
.../flink/api/table/plan/nodes/FlinkRel.scala | 43 ++
.../plan/nodes/dataset/DataSetAggregate.scala | 60 +-
.../table/plan/nodes/dataset/DataSetRel.scala | 36 --
.../nodes/datastream/DataStreamAggregate.scala | 300 +++++++++
.../api/table/plan/rules/FlinkRuleSets.scala | 1 +
.../rules/dataSet/DataSetAggregateRule.scala | 4 +-
.../datastream/DataStreamAggregateRule.scala | 78 +++
.../aggregate/AggregateAllWindowFunction.scala | 53 ++
.../AggregateReduceCombineFunction.scala | 122 ++++
.../AggregateReduceGroupFunction.scala | 108 +---
.../table/runtime/aggregate/AggregateUtil.scala | 36 +-
.../aggregate/AggregateWindowFunction.scala | 59 ++
.../runtime/aggregate/PropertyCollector.scala | 42 ++
.../table/runtime/aggregate/WindowEndRead.scala | 38 ++
.../runtime/aggregate/WindowPropertyRead.scala | 33 +
.../runtime/aggregate/WindowStartRead.scala | 38 ++
.../org/apache/flink/api/table/table.scala | 132 +++-
.../api/table/typeutils/InternalTypeInfo.scala | 81 +++
.../api/table/typeutils/IntervalTypeInfo.scala | 109 ----
.../table/typeutils/RowIntervalTypeInfo.scala | 36 ++
.../table/typeutils/TimeIntervalTypeInfo.scala | 113 ++++
.../api/table/typeutils/TypeCheckUtils.scala | 8 +-
.../api/table/typeutils/TypeCoercion.scala | 12 +-
.../table/validate/ExprValidationResult.scala | 41 --
.../api/table/validate/ValidationResult.scala | 53 ++
.../org/apache/flink/api/table/windows.scala | 390 ++++++++++++
.../api/java/batch/TableEnvironmentITCase.java | 6 +-
.../scala/batch/TableEnvironmentITCase.scala | 4 +-
.../api/scala/batch/table/CalcITCase.scala | 6 +-
.../scala/stream/table/AggregationsITCase.scala | 198 ++++++
.../scala/stream/table/GroupWindowTest.scala | 614 +++++++++++++++++++
.../scala/stream/table/UnsupportedOpsTest.scala | 8 -
.../flink/api/table/utils/TableTestBase.scala | 25 +-
72 files changed, 4198 insertions(+), 623 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/docs/dev/table_api.md
----------------------------------------------------------------------
diff --git a/docs/dev/table_api.md b/docs/dev/table_api.md
index 57a2b5c..6da2807 100644
--- a/docs/dev/table_api.md
+++ b/docs/dev/table_api.md
@@ -965,7 +965,9 @@ composite = suffixed | atom ;
suffixed = interval | cast | as | aggregation | if | functionCall ;
-interval = composite , "." , ("year" | "years" | "month" | "months" | "day" | "days" | "hour" | "hours" | "minute" | "minutes" | "second" | "seconds" | "milli" | "millis") ;
+timeInterval = composite , "." , ("year" | "years" | "month" | "months" | "day" | "days" | "hour" | "hours" | "minute" | "minutes" | "second" | "seconds" | "milli" | "millis") ;
+
+rowInterval = composite , "." , "rows" ;
cast = composite , ".cast(" , dataType , ")" ;
@@ -973,7 +975,7 @@ dataType = "BYTE" | "SHORT" | "INT" | "LONG" | "FLOAT" | "DOUBLE" | "BOOLEAN" |
as = composite , ".as(" , fieldReference , ")" ;
-aggregation = composite , ( ".sum" | ".min" | ".max" | ".count" | ".avg" ) , [ "()" ] ;
+aggregation = composite , ( ".sum" | ".min" | ".max" | ".count" | ".avg" | ".start" | ".end" ) , [ "()" ] ;
if = composite , ".?(" , expression , "," , expression , ")" ;
@@ -992,7 +994,7 @@ timePointUnit = "YEAR" | "MONTH" | "DAY" | "HOUR" | "MINUTE" | "SECOND" | "QUART
{% endhighlight %}
Here, `literal` is a valid Java literal, `fieldReference` specifies a column in the data (or all columns if `*` is used), and `functionIdentifier` specifies a supported scalar function. The
-column names and function names follow Java identifier syntax. Expressions specified as Strings can also use prefix notation instead of suffix notation to call operators and functions.
+column names and function names follow Java identifier syntax. The column name `rowtime` is a reserved logical attribute in streaming environments. Expressions specified as Strings can also use prefix notation instead of suffix notation to call operators and functions.
If working with exact numeric values or large decimals is required, the Table API also supports Java's BigDecimal type. In the Scala Table API decimals can be defined by `BigDecimal("123456")` and in Java by appending a "p" for precise e.g. `123456p`.
@@ -1002,6 +1004,266 @@ Temporal intervals can be represented as number of months (`Types.INTERVAL_MONTH
{% top %}
+### Windows
+
+The Table API is a declarative API to define queries on batch and streaming tables. Projection, selection, and union operations can be applied both on streaming and batch tables without additional semantics. Aggregations on (possibly) infinite streaming tables, however, can only be computed on finite groups of records. Group-window aggregates group rows into finite groups based on time or row-count intervals and evaluate aggregation functions once per group. For batch tables, group-windows are a convenient shortcut to group records by time intervals.
+
+Group-windows are defined using the `window(w: GroupWindow)` clause. The following example shows how to define a group-window aggregation on a table.
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+val table = input
+ .window(w: GroupWindow) // define window
+ .select("b.sum") // aggregate
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+val table = input
+ .window(w: GroupWindow) // define window
+ .select('b.sum) // aggregate
+{% endhighlight %}
+</div>
+</div>
+
+In streaming environments, group-window aggregates can only be computed in parallel, if they are *keyed*, i.e., there is an additional `groupBy` attribute. Group-window aggregates without additional `groupBy`, such as in the example above, can only be evaluated in a single, non-parallel task. The following example shows how to define a keyed group-window aggregation on a table.
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+val table = input
+ .groupBy("a")
+ .window(w: GroupWindow) // define window
+ .select("a, b.sum") // aggregate
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+val table = input
+ .groupBy('a)
+ .window(w: GroupWindow) // define window
+ .select('a, 'b.sum) // aggregate
+{% endhighlight %}
+</div>
+</div>
+
+The `GroupWindow` parameter defines how rows are mapped to windows. `GroupWindow` is not an interface that users can implement. Instead, the Table API provides a set of predefined `GroupWindow` classes with specific semantics, which are translated into underlying `DataStream` or `DataSet` operations. The supported window definitions are listed below.
+By assigning the group-window an alias using `as`, properties such as the start and end timestamp of a time window can be accessed in the `select` statement.
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+val table = input
+ .groupBy("a")
+ .window(XXX.as("myWin")) // define window alias
+ .select("a, myWin.start, myWin.end, b.count") // aggregate
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+val table = input
+ .groupBy('a)
+ .window(XXX as 'myWin) // define window alias
+ .select('a, 'myWin.start, 'myWin.end, 'b.count) // aggregate
+{% endhighlight %}
+</div>
+</div>
+
+#### Tumble (Tumbling Windows)
+
+A tumbling window assigns rows to non-overlapping, continuous windows of fixed length. For example, a tumbling window of 5 minutes groups rows in 5 minutes intervals. Tumbling windows can be defined on event-time, processing-time, or on a row-count.
+
+Tumbling windows are defined by using the `Tumble` class as follows:
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Method</th>
+ <th class="text-left" style="width: 20%">Required?</th>
+ <th class="text-left">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><code>over</code></td>
+ <td>Required.</td>
+ <td>Defines the length the window, either as time or row-count interval.</td>
+ </tr>
+ <tr>
+ <td><code>on</code></td>
+ <td>Required for streaming event-time windows and windows on batch tables.</td>
+ <td>Defines the time mode for streaming tables (<code>rowtime</code> is a logical system attribute); for batch tables, the time attribute on which records are grouped.</td>
+ </tr>
+ <tr>
+ <td><code>as</code></td>
+ <td>Optional.</td>
+ <td>Assigns an alias to the window that can be used in the following <code>select()</code> clause to access window properties such as window start or end time.</td>
+ </tr>
+ </tbody>
+</table>
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+// Tumbling Event-time Window
+.window(Tumble.over("10.minutes").on("rowtime").as("w"))
+
+// Tumbling Processing-time Window
+.window(Tumble.over("10.minutes").as("w"))
+
+// Tumbling Row-count Window
+.window(Tumble.over("10.rows").as("w"))
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+// Tumbling Event-time Window
+.window(Tumble over 10.minutes on 'rowtime as 'w)
+
+// Tumbling Processing-time Window
+.window(Tumble over 10.minutes as 'w)
+
+// Tumbling Row-count Window
+.window(Tumble over 10.rows as 'w)
+{% endhighlight %}
+</div>
+</div>
+
+#### Slide (Sliding Windows)
+
+A sliding window has a fixed size and slides by a specified slide interval. If the slide interval is smaller than the window size, sliding windows are overlapping. Thus, rows can be assigned to multiple windows. For example, a sliding window of 15 minutes size and 5 minute slide interval assigns each row to 3 different windows of 15 minute size, which are evaluated in an interval of 5 minutes. Sliding windows can be defined on event-time, processing-time, or on a row-count.
+
+Sliding windows are defined by using the `Slide` class as follows:
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Method</th>
+ <th class="text-left" style="width: 20%">Required?</th>
+ <th class="text-left">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><code>over</code></td>
+ <td>Required.</td>
+ <td>Defines the length of the window, either as time or row-count interval.</td>
+ </tr>
+ <tr>
+ <td><code>every</code></td>
+ <td>Required.</td>
+ <td>Defines the slide interval, either as time or row-count interval. The slide interval must be of the same type as the size interval.</td>
+ </tr>
+ <tr>
+ <td><code>on</code></td>
+ <td>Required for event-time windows and windows on batch tables.</td>
+ <td>Defines the time mode for streaming tables (<code>rowtime</code> is a logical system attribute); for batch tables, the time attribute on which records are grouped</td>
+ </tr>
+ <tr>
+ <td><code>as</code></td>
+ <td>Optional.</td>
+ <td>Assigns an alias to the window that can be used in the following <code>select()</code> clause to access window properties such as window start or end time.</td>
+ </tr>
+ </tbody>
+</table>
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+// Sliding Event-time Window
+.window(Slide.over("10.minutes").every("5.minutes").on("rowtime").as("w"))
+
+// Sliding Processing-time window
+.window(Slide.over("10.minutes").every("5.minutes").as("w"))
+
+// Sliding Row-count window
+.window(Slide.over("10.rows").every("5.rows").as("w"))
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+// Sliding Event-time Window
+.window(Slide over 10.minutes every 5.minutes on 'rowtime as 'w)
+
+// Sliding Processing-time window
+.window(Slide over 10.minutes every 5.minutes as 'w)
+
+// Sliding Row-count window
+.window(Slide over 10.rows every 5.rows as 'w)
+{% endhighlight %}
+</div>
+</div>
+
+#### Session (Session Windows)
+
+Session windows do not have a fixed size but their bounds are defined by an interval of inactivity, i.e., a session window is closes if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time or processing-time.
+
+A session window is defined by using the `Session` class as follows:
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Method</th>
+ <th class="text-left" style="width: 20%">Required?</th>
+ <th class="text-left">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><code>withGap</code></td>
+ <td>Required.</td>
+ <td>Defines the gap between two windows as time interval.</td>
+ </tr>
+ <tr>
+ <td><code>on</code></td>
+ <td>Required for event-time windows and windows on batch tables.</td>
+ <td>Defines the time mode for streaming tables (<code>rowtime</code> is a logical system attribute); for batch tables, the time attribute on which records are grouped</td>
+ </tr>
+ <tr>
+ <td><code>as</code></td>
+ <td>Optional.</td>
+ <td>Assigns an alias to the window that can be used in the following <code>select()</code> clause to access window properties such as window start or end time.</td>
+ </tr>
+ </tbody>
+</table>
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight java %}
+// Session Event-time Window
+.window(Session.withGap("10.minutes").on("rowtime").as("w"))
+
+// Session Processing-time Window
+.window(Session.withGap("10.minutes").as("w"))
+{% endhighlight %}
+</div>
+
+<div data-lang="scala" markdown="1">
+{% highlight scala %}
+// Session Event-time Window
+.window(Session withGap 10.minutes on 'rowtime as 'w)
+
+// Session Processing-time Window
+.window(Session withGap 10.minutes as 'w)
+{% endhighlight %}
+</div>
+</div>
+
+#### Limitations
+
+Currently the following features are not supported yet:
+
+- Row-count windows on event-time
+- Windows on batch tables
SQL
----
@@ -1549,6 +1811,83 @@ STRING.toTimestamp()
<tr>
<td>
{% highlight java %}
+NUMERIC.year
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of months for a given number of years.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.month
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of months for a given number of months.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.day
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of days.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.hour
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of hours.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.minute
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of minutes.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.second
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of seconds.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.milli
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight java %}
TEMPORAL.extract(TIMEINTERVALUNIT)
{% endhighlight %}
</td>
@@ -1656,6 +1995,17 @@ temporalOverlaps(TIMEPOINT, TEMPORAL, TIMEPOINT, TEMPORAL)
</td>
</tr>
+ <tr>
+ <td>
+ {% highlight java %}
+NUMERIC.rows
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of rows.</p>
+ </td>
+ </tr>
+
</tbody>
</table>
@@ -1989,6 +2339,83 @@ STRING.toTimestamp
<tr>
<td>
{% highlight scala %}
+NUMERIC.year
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of months for a given number of years.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.month
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of months for a given number of months.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.day
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of days.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.hour
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of hours.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.minute
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of minutes.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.second
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds for a given number of seconds.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.milli
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of milliseconds.</p>
+ </td>
+ </tr>
+
+ <tr>
+ <td>
+ {% highlight scala %}
TEMPORAL.extract(TimeIntervalUnit)
{% endhighlight %}
</td>
@@ -2096,6 +2523,17 @@ temporalOverlaps(TIMEPOINT, TEMPORAL, TIMEPOINT, TEMPORAL)
</td>
</tr>
+ <tr>
+ <td>
+ {% highlight scala %}
+NUMERIC.rows
+{% endhighlight %}
+ </td>
+ <td>
+ <p>Creates an interval of rows.</p>
+ </td>
+ </tr>
+
</tbody>
</table>
</div>
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/groupWindows.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/groupWindows.scala
new file mode 100644
index 0000000..3bbe753
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/java/table/groupWindows.scala
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.table
+
+import org.apache.flink.api.table.{SessionWindow, SlideWithSize, TumblingWindow}
+
+/**
+ * Helper class for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
+ * windows of a specified fixed length. For example, a tumbling window of 5 minutes size groups
+ * elements in 5 minutes intervals.
+ */
+object Tumble {
+
+ /**
+ * Creates a tumbling window. Tumbling windows are consecutive, non-overlapping
+ * windows of a specified fixed length. For example, a tumbling window of 5 minutes size groups
+ * elements in 5 minutes intervals.
+ *
+ * @param size the size of the window as time or row-count interval.
+ * @return a tumbling window
+ */
+ def over(size: String): TumblingWindow = new TumblingWindow(size)
+}
+
+/**
+ * Helper class for creating a sliding window. Sliding windows have a fixed size and slide by
+ * a specified slide interval. If the slide interval is smaller than the window size, sliding
+ * windows are overlapping. Thus, an element can be assigned to multiple windows.
+ *
+ * For example, a sliding window of size 15 minutes with 5 minutes sliding interval groups elements
+ * of 15 minutes and evaluates every five minutes. Each element is contained in three consecutive
+ * window evaluations.
+ */
+object Slide {
+
+ /**
+ * Creates a sliding window. Sliding windows have a fixed size and slide by
+ * a specified slide interval. If the slide interval is smaller than the window size, sliding
+ * windows are overlapping. Thus, an element can be assigned to multiple windows.
+ *
+ * For example, a sliding window of size 15 minutes with 5 minutes sliding interval groups
+ * elements of 15 minutes and evaluates every five minutes. Each element is contained in three
+ * consecutive window evaluations.
+ *
+ * @param size the size of the window as time or row-count interval
+ * @return a partially specified sliding window
+ */
+ def over(size: String): SlideWithSize = new SlideWithSize(size)
+}
+
+/**
+ * Helper class for creating a session window. The boundary of session windows are defined by
+ * intervals of inactivity, i.e., a session window is closes if no event appears for a defined
+ * gap period.
+ */
+object Session {
+
+ /**
+ * Creates a session window. The boundary of session windows are defined by
+ * intervals of inactivity, i.e., a session window is closes if no event appears for a defined
+ * gap period.
+ *
+ * @param gap specifies how long (as interval of milliseconds) to wait for new data before
+ * closing the session window.
+ * @return a session window
+ */
+ def withGap(gap: String): SessionWindow = new SessionWindow(gap)
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/expressionDsl.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/expressionDsl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/expressionDsl.scala
index 5df9175..836db3e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/expressionDsl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/expressionDsl.scala
@@ -21,7 +21,7 @@ import java.sql.{Date, Time, Timestamp}
import org.apache.calcite.avatica.util.DateTimeUtils._
import org.apache.flink.api.common.typeinfo.{SqlTimeTypeInfo, TypeInformation}
-import org.apache.flink.api.table.expressions.ExpressionUtils.{toMilliInterval, toMonthInterval}
+import org.apache.flink.api.table.expressions.ExpressionUtils.{toMilliInterval, toMonthInterval, toRowInterval}
import org.apache.flink.api.table.expressions.TimeIntervalUnit.TimeIntervalUnit
import org.apache.flink.api.table.expressions._
@@ -103,6 +103,16 @@ trait ImplicitExpressionOperations {
def desc = Desc(expr)
/**
+ * Returns the start time of a window when applied on a window reference.
+ */
+ def start = WindowStart(expr)
+
+ /**
+ * Returns the end time of a window when applied on a window reference.
+ */
+ def end = WindowEnd(expr)
+
+ /**
* Ternary conditional operator that decides which of two other expressions should be evaluated
* based on a evaluated boolean condition.
*
@@ -356,7 +366,7 @@ trait ImplicitExpressionOperations {
*/
def days = day
- /**
+ /**
* Creates an interval of the given number of hours.
*
* @return interval of milliseconds
@@ -370,7 +380,7 @@ trait ImplicitExpressionOperations {
*/
def hours = hour
- /**
+ /**
* Creates an interval of the given number of minutes.
*
* @return interval of milliseconds
@@ -384,7 +394,7 @@ trait ImplicitExpressionOperations {
*/
def minutes = minute
- /**
+ /**
* Creates an interval of the given number of seconds.
*
* @return interval of milliseconds
@@ -398,7 +408,7 @@ trait ImplicitExpressionOperations {
*/
def seconds = second
- /**
+ /**
* Creates an interval of the given number of milliseconds.
*
* @return interval of milliseconds
@@ -411,6 +421,16 @@ trait ImplicitExpressionOperations {
* @return interval of milliseconds
*/
def millis = milli
+
+ // row interval type
+
+ /**
+ * Creates an interval of rows.
+ *
+ * @return interval of rows
+ */
+ def rows = toRowInterval(expr)
+
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/groupWindows.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/groupWindows.scala
new file mode 100644
index 0000000..16fda5b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/scala/table/groupWindows.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.table
+
+import org.apache.flink.api.table.expressions.Expression
+import org.apache.flink.api.table.{SessionWindow, SlideWithSize, TumblingWindow}
+
+/**
+ * Helper object for creating a tumbling window. Tumbling windows are consecutive, non-overlapping
+ * windows of a specified fixed length. For example, a tumbling window of 5 minutes size groups
+ * elements in 5 minutes intervals.
+ */
+object Tumble {
+
+ /**
+ * Creates a tumbling window. Tumbling windows are fixed-size, consecutive, non-overlapping
+ * windows. For example, a tumbling window of 5 minutes size groups
+ * elements in 5 minutes intervals.
+ *
+ * @param size the size of the window as time or row-count interval.
+ * @return a tumbling window
+ */
+ def over(size: Expression): TumblingWindow = new TumblingWindow(size)
+}
+
+/**
+ * Helper object for creating a sliding window. Sliding windows have a fixed size and slide by
+ * a specified slide interval. If the slide interval is smaller than the window size, sliding
+ * windows are overlapping. Thus, an element can be assigned to multiple windows.
+ *
+ * For example, a sliding window of size 15 minutes with 5 minutes sliding interval groups elements
+ * of 15 minutes and evaluates every five minutes. Each element is contained in three consecutive
+ * window evaluations.
+ */
+object Slide {
+
+ /**
+ * Creates a sliding window. Sliding windows have a fixed size and slide by
+ * a specified slide interval. If the slide interval is smaller than the window size, sliding
+ * windows are overlapping. Thus, an element can be assigned to multiple windows.
+ *
+ * For example, a sliding window of size 15 minutes with 5 minutes sliding interval groups
+ * elements of 15 minutes and evaluates every five minutes. Each element is contained in three
+ * consecutive
+ *
+ * @param size the size of the window as time or row-count interval
+ * @return a partially specified sliding window
+ */
+ def over(size: Expression): SlideWithSize = new SlideWithSize(size)
+}
+
+/**
+ * Helper object for creating a session window. The boundary of session windows are defined by
+ * intervals of inactivity, i.e., a session window is closes if no event appears for a defined
+ * gap period.
+ */
+object Session {
+
+ /**
+ * Creates a session window. The boundary of session windows are defined by
+ * intervals of inactivity, i.e., a session window is closes if no event appears for a defined
+ * gap period.
+ *
+ * @param gap specifies how long (as interval of milliseconds) to wait for new data before
+ * closing the session window.
+ * @return a session window
+ */
+ def withGap(gap: Expression): SessionWindow = new SessionWindow(gap)
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
index 1d34777..24b385c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/BatchTableEnvironment.scala
@@ -73,7 +73,7 @@ abstract class BatchTableEnvironment(
val m = internalNamePattern.findFirstIn(name)
m match {
case Some(_) =>
- throw new ValidationException(s"Illegal Table name. " +
+ throw new TableException(s"Illegal Table name. " +
s"Please choose a name that does not contain the pattern $internalNamePattern")
case None =>
}
@@ -96,7 +96,7 @@ abstract class BatchTableEnvironment(
if (isRegistered(tableName)) {
new Table(this, CatalogNode(tableName, getRowType(tableName)))
} else {
- throw new ValidationException(s"Table \'$tableName\' was not found in the registry.")
+ throw new TableException(s"Table \'$tableName\' was not found in the registry.")
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkRelBuilder.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkRelBuilder.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkRelBuilder.scala
index 1215806..ea4eed0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkRelBuilder.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkRelBuilder.scala
@@ -20,12 +20,20 @@ package org.apache.flink.api.table
import java.util.Collections
+import org.apache.calcite.plan.volcano.VolcanoPlanner
+import java.lang.Iterable
+
import org.apache.calcite.jdbc.CalciteSchema
import org.apache.calcite.plan._
-import org.apache.calcite.plan.volcano.VolcanoPlanner
import org.apache.calcite.prepare.CalciteCatalogReader
+import org.apache.calcite.rel.logical.LogicalAggregate
import org.apache.calcite.rex.RexBuilder
+import org.apache.calcite.tools.RelBuilder.{AggCall, GroupKey}
import org.apache.calcite.tools.{FrameworkConfig, RelBuilder}
+import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
+import org.apache.flink.api.table.expressions.WindowProperty
+import org.apache.flink.api.table.plan.logical.LogicalWindow
+import org.apache.flink.api.table.plan.logical.rel.LogicalWindowAggregate
/**
* Flink specific [[RelBuilder]] that changes the default type factory to a [[FlinkTypeFactory]].
@@ -41,10 +49,25 @@ class FlinkRelBuilder(
def getPlanner: RelOptPlanner = cluster.getPlanner
- def getCluster = cluster
+ def getCluster: RelOptCluster = relOptCluster
override def getTypeFactory: FlinkTypeFactory =
super.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+
+ def aggregate(
+ window: LogicalWindow,
+ groupKey: GroupKey,
+ namedProperties: Seq[NamedWindowProperty],
+ aggCalls: Iterable[AggCall])
+ : RelBuilder = {
+ // build logical aggregate
+ val aggregate = super.aggregate(groupKey, aggCalls).build().asInstanceOf[LogicalAggregate]
+
+ // build logical window aggregate from it
+ push(LogicalWindowAggregate.create(window, namedProperties, aggregate))
+ this
+ }
+
}
object FlinkRelBuilder {
@@ -69,4 +92,11 @@ object FlinkRelBuilder {
new FlinkRelBuilder(config.getContext, cluster, relOptSchema)
}
+ /**
+ * Information necessary to create a window aggregate.
+ *
+ * Similar to [[RelBuilder.AggCall]] or [[RelBuilder.GroupKey]].
+ */
+ case class NamedWindowProperty(name: String, property: WindowProperty)
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkTypeFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkTypeFactory.scala
index 77eb907..1f607e4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/FlinkTypeFactory.scala
@@ -30,7 +30,7 @@ import org.apache.flink.api.common.typeinfo.{SqlTimeTypeInfo, TypeInformation}
import org.apache.flink.api.java.typeutils.ValueTypeInfo._
import org.apache.flink.api.table.FlinkTypeFactory.typeInfoToSqlTypeName
import org.apache.flink.api.table.plan.schema.GenericRelDataType
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.TimeIntervalTypeInfo
import org.apache.flink.api.table.typeutils.TypeCheckUtils.isSimple
import scala.collection.mutable
@@ -106,8 +106,8 @@ object FlinkTypeFactory {
case SqlTimeTypeInfo.DATE => DATE
case SqlTimeTypeInfo.TIME => TIME
case SqlTimeTypeInfo.TIMESTAMP => TIMESTAMP
- case IntervalTypeInfo.INTERVAL_MONTHS => INTERVAL_YEAR_MONTH
- case IntervalTypeInfo.INTERVAL_MILLIS => INTERVAL_DAY_SECOND
+ case TimeIntervalTypeInfo.INTERVAL_MONTHS => INTERVAL_YEAR_MONTH
+ case TimeIntervalTypeInfo.INTERVAL_MILLIS => INTERVAL_DAY_SECOND
case CHAR_TYPE_INFO | CHAR_VALUE_TYPE_INFO =>
throw TableException("Character type is not supported.")
@@ -131,8 +131,8 @@ object FlinkTypeFactory {
case DATE => SqlTimeTypeInfo.DATE
case TIME => SqlTimeTypeInfo.TIME
case TIMESTAMP => SqlTimeTypeInfo.TIMESTAMP
- case typeName if YEAR_INTERVAL_TYPES.contains(typeName) => IntervalTypeInfo.INTERVAL_MONTHS
- case typeName if DAY_INTERVAL_TYPES.contains(typeName) => IntervalTypeInfo.INTERVAL_MILLIS
+ case typeName if YEAR_INTERVAL_TYPES.contains(typeName) => TimeIntervalTypeInfo.INTERVAL_MONTHS
+ case typeName if DAY_INTERVAL_TYPES.contains(typeName) => TimeIntervalTypeInfo.INTERVAL_MILLIS
case NULL =>
throw TableException("Type NULL is not supported. " +
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
index ac21834..b9e889d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/StreamTableEnvironment.scala
@@ -81,6 +81,47 @@ abstract class StreamTableEnvironment(
protected def createUniqueTableName(): String = "_DataStreamTable_" + nameCntr.getAndIncrement()
/**
+ * Returns field names and field positions for a given [[TypeInformation]].
+ *
+ * Field names are automatically extracted for
+ * [[org.apache.flink.api.common.typeutils.CompositeType]].
+ * The method fails if inputType is not a
+ * [[org.apache.flink.api.common.typeutils.CompositeType]].
+ *
+ * @param inputType The TypeInformation extract the field names and positions from.
+ * @tparam A The type of the TypeInformation.
+ * @return A tuple of two arrays holding the field names and corresponding field positions.
+ */
+ override protected[flink] def getFieldInfo[A](inputType: TypeInformation[A])
+ : (Array[String], Array[Int]) = {
+ val fieldInfo = super.getFieldInfo(inputType)
+ if (fieldInfo._1.contains("rowtime")) {
+ throw new TableException("'rowtime' ia a reserved field name in stream environment.")
+ }
+ fieldInfo
+ }
+
+ /**
+ * Returns field names and field positions for a given [[TypeInformation]] and [[Array]] of
+ * [[Expression]].
+ *
+ * @param inputType The [[TypeInformation]] against which the [[Expression]]s are evaluated.
+ * @param exprs The expressions that define the field names.
+ * @tparam A The type of the TypeInformation.
+ * @return A tuple of two arrays holding the field names and corresponding field positions.
+ */
+ override protected[flink] def getFieldInfo[A](
+ inputType: TypeInformation[A],
+ exprs: Array[Expression])
+ : (Array[String], Array[Int]) = {
+ val fieldInfo = super.getFieldInfo(inputType, exprs)
+ if (fieldInfo._1.contains("rowtime")) {
+ throw new TableException("'rowtime' is a reserved field name in stream environment.")
+ }
+ fieldInfo
+ }
+
+ /**
* Ingests a registered table and returns the resulting [[Table]].
*
* The table to ingest must be registered in the [[TableEnvironment]]'s catalog.
@@ -226,11 +267,11 @@ abstract class StreamTableEnvironment(
}
catch {
case e: CannotPlanException =>
- throw new TableException(
+ throw TableException(
s"Cannot generate a valid execution plan for the given query: \n\n" +
s"${RelOptUtil.toString(relNode)}\n" +
s"This exception indicates that the query uses an unsupported SQL feature.\n" +
- s"Please check the documentation for the set of currently supported SQL features.")
+ s"Please check the documentation for the set of currently supported SQL features.", e)
}
dataStreamPlan
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
index df97d2d..e8734f5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/TableEnvironment.scala
@@ -180,7 +180,7 @@ abstract class TableEnvironment(val config: TableConfig) {
// check that table belongs to this table environment
if (table.tableEnv != this) {
- throw new ValidationException(
+ throw new TableException(
"Only tables that belong to this TableEnvironment can be registered.")
}
@@ -230,7 +230,7 @@ abstract class TableEnvironment(val config: TableConfig) {
*
* @param name The name under which the table is registered.
* @param table The table to register in the catalog
- * @throws ValidationException if another table is registered under the provided name.
+ * @throws TableException if another table is registered under the provided name.
*/
@throws[TableException]
protected def registerTableInternal(name: String, table: AbstractTable): Unit = {
@@ -330,7 +330,7 @@ abstract class TableEnvironment(val config: TableConfig) {
val fieldIndexes = fieldNames.indices.toArray
if (fieldNames.contains("*")) {
- throw new ValidationException("Field name can not be '*'.")
+ throw new TableException("Field name can not be '*'.")
}
(fieldNames, fieldIndexes)
@@ -408,7 +408,7 @@ abstract class TableEnvironment(val config: TableConfig) {
val (fieldIndexes, fieldNames) = indexedNames.unzip
if (fieldNames.contains("*")) {
- throw new ValidationException("Field name can not be '*'.")
+ throw new TableException("Field name can not be '*'.")
}
(fieldNames.toArray, fieldIndexes.toArray)
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/Types.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/Types.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/Types.scala
index 64d4612..a988152 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/Types.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/Types.scala
@@ -18,7 +18,7 @@
package org.apache.flink.api.table
import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo}
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.TimeIntervalTypeInfo
/**
* This class enumerates all supported types of the Table API.
@@ -39,7 +39,7 @@ object Types {
val DATE = SqlTimeTypeInfo.DATE
val TIME = SqlTimeTypeInfo.TIME
val TIMESTAMP = SqlTimeTypeInfo.TIMESTAMP
- val INTERVAL_MONTHS = IntervalTypeInfo.INTERVAL_MONTHS
- val INTERVAL_MILLIS = IntervalTypeInfo.INTERVAL_MILLIS
+ val INTERVAL_MONTHS = TimeIntervalTypeInfo.INTERVAL_MONTHS
+ val INTERVAL_MILLIS = TimeIntervalTypeInfo.INTERVAL_MILLIS
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenUtils.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenUtils.scala
index 76f9b02..b78012c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/CodeGenUtils.scala
@@ -28,7 +28,7 @@ import org.apache.flink.api.common.typeinfo.{FractionalTypeInfo, SqlTimeTypeInfo
import org.apache.flink.api.common.typeutils.CompositeType
import org.apache.flink.api.java.typeutils.{PojoTypeInfo, TupleTypeInfo, TypeExtractor}
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.table.typeutils.{IntervalTypeInfo, RowTypeInfo, TypeCheckUtils}
+import org.apache.flink.api.table.typeutils.{TimeIntervalTypeInfo, RowTypeInfo, TypeCheckUtils}
object CodeGenUtils {
@@ -71,8 +71,8 @@ object CodeGenUtils {
case SqlTimeTypeInfo.TIMESTAMP => "long"
// internal primitive representation of time intervals
- case IntervalTypeInfo.INTERVAL_MONTHS => "int"
- case IntervalTypeInfo.INTERVAL_MILLIS => "long"
+ case TimeIntervalTypeInfo.INTERVAL_MONTHS => "int"
+ case TimeIntervalTypeInfo.INTERVAL_MILLIS => "long"
case _ =>
tpe.getTypeClass.getCanonicalName
@@ -106,8 +106,8 @@ object CodeGenUtils {
case CHAR_TYPE_INFO => "'\\0'"
case SqlTimeTypeInfo.DATE | SqlTimeTypeInfo.TIME => "-1"
case SqlTimeTypeInfo.TIMESTAMP => "-1L"
- case IntervalTypeInfo.INTERVAL_MONTHS => "-1"
- case IntervalTypeInfo.INTERVAL_MILLIS => "-1L"
+ case TimeIntervalTypeInfo.INTERVAL_MONTHS => "-1"
+ case TimeIntervalTypeInfo.INTERVAL_MILLIS => "-1L"
case _ => "null"
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarOperators.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarOperators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarOperators.scala
index 094a224..bf76015 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarOperators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/codegen/calls/ScalarOperators.scala
@@ -24,7 +24,7 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
import org.apache.flink.api.common.typeinfo.{NumericTypeInfo, SqlTimeTypeInfo, TypeInformation}
import org.apache.flink.api.table.codegen.CodeGenUtils._
import org.apache.flink.api.table.codegen.{CodeGenException, GeneratedExpression}
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.TimeIntervalTypeInfo
import org.apache.flink.api.table.typeutils.TypeCheckUtils._
object ScalarOperators {
@@ -432,7 +432,7 @@ object ScalarOperators {
}
// Interval Months -> String
- case (IntervalTypeInfo.INTERVAL_MONTHS, STRING_TYPE_INFO) =>
+ case (TimeIntervalTypeInfo.INTERVAL_MONTHS, STRING_TYPE_INFO) =>
val method = qualifyMethod(BuiltInMethod.INTERVAL_YEAR_MONTH_TO_STRING.method)
val timeUnitRange = qualifyEnum(TimeUnitRange.YEAR_TO_MONTH)
generateUnaryOperatorIfNotNull(nullCheck, targetType, operand) {
@@ -440,7 +440,7 @@ object ScalarOperators {
}
// Interval Millis -> String
- case (IntervalTypeInfo.INTERVAL_MILLIS, STRING_TYPE_INFO) =>
+ case (TimeIntervalTypeInfo.INTERVAL_MILLIS, STRING_TYPE_INFO) =>
val method = qualifyMethod(BuiltInMethod.INTERVAL_DAY_TIME_TO_STRING.method)
val timeUnitRange = qualifyEnum(TimeUnitRange.DAY_TO_SECOND)
generateUnaryOperatorIfNotNull(nullCheck, targetType, operand) {
@@ -573,17 +573,17 @@ object ScalarOperators {
(INT_TYPE_INFO, SqlTimeTypeInfo.DATE) |
(INT_TYPE_INFO, SqlTimeTypeInfo.TIME) |
(LONG_TYPE_INFO, SqlTimeTypeInfo.TIMESTAMP) |
- (INT_TYPE_INFO, IntervalTypeInfo.INTERVAL_MONTHS) |
- (LONG_TYPE_INFO, IntervalTypeInfo.INTERVAL_MILLIS) |
- (IntervalTypeInfo.INTERVAL_MONTHS, INT_TYPE_INFO) |
- (IntervalTypeInfo.INTERVAL_MILLIS, LONG_TYPE_INFO) =>
+ (INT_TYPE_INFO, TimeIntervalTypeInfo.INTERVAL_MONTHS) |
+ (LONG_TYPE_INFO, TimeIntervalTypeInfo.INTERVAL_MILLIS) |
+ (TimeIntervalTypeInfo.INTERVAL_MONTHS, INT_TYPE_INFO) |
+ (TimeIntervalTypeInfo.INTERVAL_MILLIS, LONG_TYPE_INFO) =>
internalExprCasting(operand, targetType)
// internal reinterpretation of temporal types
// Date, Time, Interval Months -> Long
case (SqlTimeTypeInfo.DATE, LONG_TYPE_INFO)
| (SqlTimeTypeInfo.TIME, LONG_TYPE_INFO)
- | (IntervalTypeInfo.INTERVAL_MONTHS, LONG_TYPE_INFO) =>
+ | (TimeIntervalTypeInfo.INTERVAL_MONTHS, LONG_TYPE_INFO) =>
internalExprCasting(operand, targetType)
case (from, to) =>
@@ -659,11 +659,11 @@ object ScalarOperators {
val operator = if (plus) "+" else "-"
(left.resultType, right.resultType) match {
- case (l: IntervalTypeInfo[_], r: IntervalTypeInfo[_]) if l == r =>
+ case (l: TimeIntervalTypeInfo[_], r: TimeIntervalTypeInfo[_]) if l == r =>
generateArithmeticOperator(operator, nullCheck, l, left, right)
- case (SqlTimeTypeInfo.DATE, IntervalTypeInfo.INTERVAL_MILLIS) |
- (IntervalTypeInfo.INTERVAL_MILLIS, SqlTimeTypeInfo.DATE) =>
+ case (SqlTimeTypeInfo.DATE, TimeIntervalTypeInfo.INTERVAL_MILLIS) |
+ (TimeIntervalTypeInfo.INTERVAL_MILLIS, SqlTimeTypeInfo.DATE) =>
generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.DATE, left, right) {
if (isTimePoint(left.resultType)) {
(leftTerm, rightTerm) => s"$leftTerm + ((int) ($rightTerm / ${MILLIS_PER_DAY}L))"
@@ -672,8 +672,8 @@ object ScalarOperators {
}
}
- case (SqlTimeTypeInfo.TIME, IntervalTypeInfo.INTERVAL_MILLIS) |
- (IntervalTypeInfo.INTERVAL_MILLIS, SqlTimeTypeInfo.TIME) =>
+ case (SqlTimeTypeInfo.TIME, TimeIntervalTypeInfo.INTERVAL_MILLIS) |
+ (TimeIntervalTypeInfo.INTERVAL_MILLIS, SqlTimeTypeInfo.TIME) =>
generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.TIME, left, right) {
if (isTimePoint(left.resultType)) {
(leftTerm, rightTerm) => s"$leftTerm + ((int) ($rightTerm))"
@@ -682,7 +682,7 @@ object ScalarOperators {
}
}
- case (SqlTimeTypeInfo.TIMESTAMP, IntervalTypeInfo.INTERVAL_MILLIS) =>
+ case (SqlTimeTypeInfo.TIMESTAMP, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
generateOperatorIfNotNull(nullCheck, SqlTimeTypeInfo.TIMESTAMP, left, right) {
(leftTerm, rightTerm) => s"$leftTerm + $rightTerm"
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/exceptions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/exceptions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/exceptions.scala
index 773e256..0a100dd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/exceptions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/exceptions.scala
@@ -38,7 +38,18 @@ case class SqlParserException(
/**
* General Exception for all errors during table handling.
*/
-case class TableException(msg: String) extends RuntimeException(msg)
+case class TableException(
+ msg: String,
+ cause: Throwable)
+ extends RuntimeException(msg, cause) {
+
+ def this(msg: String) = this(msg, null)
+
+}
+
+object TableException {
+ def apply(msg: String): TableException = new TableException(msg)
+}
/**
* Exception for all errors occurring during validation phase.
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/Expression.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/Expression.scala
index 9a824e8..c284bd3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/Expression.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/Expression.scala
@@ -22,7 +22,7 @@ import org.apache.calcite.tools.RelBuilder
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.table.trees.TreeNode
-import org.apache.flink.api.table.validate.{ExprValidationResult, ValidationSuccess}
+import org.apache.flink.api.table.validate.{ValidationResult, ValidationSuccess}
abstract class Expression extends TreeNode[Expression] {
/**
@@ -44,7 +44,7 @@ abstract class Expression extends TreeNode[Expression] {
* or `ValidationFailure` with supplement message explaining the error.
* Note: we should only call this method until `childrenValid == true`
*/
- private[flink] def validateInput(): ExprValidationResult = ValidationSuccess
+ private[flink] def validateInput(): ValidationResult = ValidationSuccess
/**
* Convert Expression to its counterpart in Calcite, i.e. RexNode
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionParser.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionParser.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionParser.scala
index 424d86c..a438c1c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionParser.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionParser.scala
@@ -24,7 +24,7 @@ import org.apache.flink.api.table.expressions.ExpressionUtils.{toMilliInterval,
import org.apache.flink.api.table.expressions.TimeIntervalUnit.TimeIntervalUnit
import org.apache.flink.api.table.expressions.TimePointUnit.TimePointUnit
import org.apache.flink.api.table.expressions.TrimMode.TrimMode
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.TimeIntervalTypeInfo
import scala.language.implicitConversions
import scala.util.parsing.combinator.{JavaTokenParsers, PackratParsers}
@@ -54,6 +54,8 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
lazy val MIN: Keyword = Keyword("min")
lazy val MAX: Keyword = Keyword("max")
lazy val SUM: Keyword = Keyword("sum")
+ lazy val START: Keyword = Keyword("start")
+ lazy val END: Keyword = Keyword("end")
lazy val CAST: Keyword = Keyword("cast")
lazy val NULL: Keyword = Keyword("Null")
lazy val IF: Keyword = Keyword("?")
@@ -80,11 +82,12 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
lazy val SECOND: Keyword = Keyword("second")
lazy val MILLIS: Keyword = Keyword("millis")
lazy val MILLI: Keyword = Keyword("milli")
+ lazy val ROWS: Keyword = Keyword("rows")
lazy val STAR: Keyword = Keyword("*")
def functionIdent: ExpressionParser.Parser[String] =
not(AS) ~ not(COUNT) ~ not(AVG) ~ not(MIN) ~ not(MAX) ~
- not(SUM) ~ not(CAST) ~ not(NULL) ~
+ not(SUM) ~ not(START) ~ not(END)~ not(CAST) ~ not(NULL) ~
not(IF) ~> super.ident
// symbols
@@ -107,10 +110,10 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
"BYTE" ^^ { ti => BasicTypeInfo.BYTE_TYPE_INFO } |
"SHORT" ^^ { ti => BasicTypeInfo.SHORT_TYPE_INFO } |
"INTERVAL_MONTHS" ^^ {
- ti => IntervalTypeInfo.INTERVAL_MONTHS.asInstanceOf[TypeInformation[_]]
+ ti => TimeIntervalTypeInfo.INTERVAL_MONTHS.asInstanceOf[TypeInformation[_]]
} |
"INTERVAL_MILLIS" ^^ {
- ti => IntervalTypeInfo.INTERVAL_MILLIS.asInstanceOf[TypeInformation[_]]
+ ti => TimeIntervalTypeInfo.INTERVAL_MILLIS.asInstanceOf[TypeInformation[_]]
} |
"INT" ^^ { ti => BasicTypeInfo.INT_TYPE_INFO } |
"LONG" ^^ { ti => BasicTypeInfo.LONG_TYPE_INFO } |
@@ -187,6 +190,12 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
lazy val suffixAvg: PackratParser[Expression] =
composite <~ "." ~ AVG ~ opt("()") ^^ { e => Avg(e) }
+ lazy val suffixStart: PackratParser[Expression] =
+ composite <~ "." ~ START ~ opt("()") ^^ { e => WindowStart(e) }
+
+ lazy val suffixEnd: PackratParser[Expression] =
+ composite <~ "." ~ END ~ opt("()") ^^ { e => WindowEnd(e) }
+
lazy val suffixCast: PackratParser[Expression] =
composite ~ "." ~ CAST ~ "(" ~ dataType ~ ")" ^^ {
case e ~ _ ~ _ ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -265,11 +274,14 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
case expr ~ _ ~ (MILLIS.key | MILLI.key)=> toMilliInterval(expr, 1)
}
+ lazy val suffixRowInterval : PackratParser[Expression] =
+ composite <~ "." ~ ROWS ^^ { e => ExpressionUtils.toRowInterval(e) }
+
lazy val suffixed: PackratParser[Expression] =
- suffixTimeInterval | suffixSum | suffixMin | suffixMax |
- suffixCount | suffixAvg | suffixCast | suffixAs | suffixTrim | suffixTrimWithoutArgs |
- suffixIf | suffixAsc | suffixDesc | suffixToDate | suffixToTimestamp | suffixToTime |
- suffixExtract | suffixFloor | suffixCeil |
+ suffixTimeInterval | suffixRowInterval | suffixSum | suffixMin | suffixMax | suffixStart |
+ suffixEnd | suffixCount | suffixAvg | suffixCast | suffixAs | suffixTrim |
+ suffixTrimWithoutArgs | suffixIf | suffixAsc | suffixDesc | suffixToDate |
+ suffixToTimestamp | suffixToTime | suffixExtract | suffixFloor | suffixCeil |
suffixFunctionCall | suffixFunctionCallOneArg // function call must always be at the end
// prefix operators
@@ -289,6 +301,12 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
lazy val prefixAvg: PackratParser[Expression] =
AVG ~ "(" ~> expression <~ ")" ^^ { e => Avg(e) }
+ lazy val prefixStart: PackratParser[Expression] =
+ START ~ "(" ~> expression <~ ")" ^^ { e => WindowStart(e) }
+
+ lazy val prefixEnd: PackratParser[Expression] =
+ END ~ "(" ~> expression <~ ")" ^^ { e => WindowEnd(e) }
+
lazy val prefixCast: PackratParser[Expression] =
CAST ~ "(" ~ expression ~ "," ~ dataType ~ ")" ^^ {
case _ ~ _ ~ e ~ _ ~ dt ~ _ => Cast(e, dt)
@@ -333,7 +351,7 @@ object ExpressionParser extends JavaTokenParsers with PackratParsers {
}
lazy val prefixed: PackratParser[Expression] =
- prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg |
+ prefixSum | prefixMin | prefixMax | prefixCount | prefixAvg | prefixStart | prefixEnd |
prefixCast | prefixAs | prefixTrim | prefixTrimWithoutArgs | prefixIf | prefixExtract |
prefixFloor | prefixCeil |
prefixFunctionCall | prefixFunctionCallOneArg // function call must always be at the end
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionUtils.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionUtils.scala
index 6ac9f58..c071c59 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ExpressionUtils.scala
@@ -25,22 +25,33 @@ import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rex.{RexBuilder, RexNode}
import org.apache.calcite.sql.fun.SqlStdOperatorTable
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
object ExpressionUtils {
private[flink] def toMonthInterval(expr: Expression, multiplier: Int): Expression = expr match {
case Literal(value: Int, BasicTypeInfo.INT_TYPE_INFO) =>
- Literal(value * multiplier, IntervalTypeInfo.INTERVAL_MONTHS)
+ Literal(value * multiplier, TimeIntervalTypeInfo.INTERVAL_MONTHS)
case _ =>
- Cast(Mul(expr, Literal(multiplier)), IntervalTypeInfo.INTERVAL_MONTHS)
+ Cast(Mul(expr, Literal(multiplier)), TimeIntervalTypeInfo.INTERVAL_MONTHS)
}
private[flink] def toMilliInterval(expr: Expression, multiplier: Long): Expression = expr match {
case Literal(value: Int, BasicTypeInfo.INT_TYPE_INFO) =>
- Literal(value * multiplier, IntervalTypeInfo.INTERVAL_MILLIS)
+ Literal(value * multiplier, TimeIntervalTypeInfo.INTERVAL_MILLIS)
+ case Literal(value: Long, BasicTypeInfo.LONG_TYPE_INFO) =>
+ Literal(value * multiplier, TimeIntervalTypeInfo.INTERVAL_MILLIS)
case _ =>
- Cast(Mul(expr, Literal(multiplier)), IntervalTypeInfo.INTERVAL_MILLIS)
+ Cast(Mul(expr, Literal(multiplier)), TimeIntervalTypeInfo.INTERVAL_MILLIS)
+ }
+
+ private[flink] def toRowInterval(expr: Expression): Expression = expr match {
+ case Literal(value: Int, BasicTypeInfo.INT_TYPE_INFO) =>
+ Literal(value.toLong, RowIntervalTypeInfo.INTERVAL_ROWS)
+ case Literal(value: Long, BasicTypeInfo.LONG_TYPE_INFO) =>
+ Literal(value, RowIntervalTypeInfo.INTERVAL_ROWS)
+ case _ =>
+ throw new IllegalArgumentException("Invalid value for row interval literal.")
}
// ----------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/InputTypeSpec.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/InputTypeSpec.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/InputTypeSpec.scala
index 7df46c2..f545d13 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/InputTypeSpec.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/InputTypeSpec.scala
@@ -38,7 +38,7 @@ trait InputTypeSpec extends Expression {
*/
private[flink] def expectedTypes: Seq[TypeInformation[_]]
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
val typeMismatches = mutable.ArrayBuffer.empty[String]
children.zip(expectedTypes).zipWithIndex.foreach { case ((e, tpe), i) =>
if (e.resultType != tpe) {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/aggregations.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/aggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/aggregations.scala
index 8cc56d8..259f7e5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/aggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/aggregations.scala
@@ -21,7 +21,6 @@ import org.apache.calcite.rex.RexNode
import org.apache.calcite.sql.fun.SqlStdOperatorTable
import org.apache.calcite.tools.RelBuilder
import org.apache.calcite.tools.RelBuilder.AggCall
-
import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.api.table.typeutils.TypeCheckUtils
@@ -47,7 +46,7 @@ case class Sum(child: Expression) extends Aggregation {
override private[flink] def resultType = child.resultType
- override private[flink] def validateInput =
+ override private[flink] def validateInput() =
TypeCheckUtils.assertNumericExpr(child.resultType, "sum")
}
@@ -60,7 +59,7 @@ case class Min(child: Expression) extends Aggregation {
override private[flink] def resultType = child.resultType
- override private[flink] def validateInput =
+ override private[flink] def validateInput() =
TypeCheckUtils.assertOrderableExpr(child.resultType, "min")
}
@@ -73,7 +72,7 @@ case class Max(child: Expression) extends Aggregation {
override private[flink] def resultType = child.resultType
- override private[flink] def validateInput =
+ override private[flink] def validateInput() =
TypeCheckUtils.assertOrderableExpr(child.resultType, "max")
}
@@ -96,6 +95,6 @@ case class Avg(child: Expression) extends Aggregation {
override private[flink] def resultType = child.resultType
- override private[flink] def validateInput =
+ override private[flink] def validateInput() =
TypeCheckUtils.assertNumericExpr(child.resultType, "avg")
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/arithmetic.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/arithmetic.scala
index 4a7978a..cf4f82a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/arithmetic.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/arithmetic.scala
@@ -44,7 +44,7 @@ abstract class BinaryArithmetic extends BinaryExpression {
}
// TODO: tighten this rule once we implemented type coercion rules during validation
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!isNumeric(left.resultType) || !isNumeric(right.resultType)) {
ValidationFailure(s"$this requires both operands Numeric, get " +
s"$left : ${left.resultType} and $right : ${right.resultType}")
@@ -81,7 +81,7 @@ case class Plus(left: Expression, right: Expression) extends BinaryArithmetic {
}
}
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (isString(left.resultType) || isString(right.resultType)) {
ValidationSuccess
} else if (isTimeInterval(left.resultType) && left.resultType == right.resultType) {
@@ -110,7 +110,7 @@ case class UnaryMinus(child: Expression) extends UnaryExpression {
override private[flink] def resultType = child.resultType
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (isNumeric(child.resultType)) {
ValidationSuccess
} else if (isTimeInterval(child.resultType)) {
@@ -126,7 +126,7 @@ case class Minus(left: Expression, right: Expression) extends BinaryArithmetic {
private[flink] val sqlOperator = SqlStdOperatorTable.MINUS
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (isTimeInterval(left.resultType) && left.resultType == right.resultType) {
ValidationSuccess
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/call.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/call.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/call.scala
index 2c64136..39367be 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/call.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/call.scala
@@ -21,7 +21,7 @@ import org.apache.calcite.rex.RexNode
import org.apache.calcite.tools.RelBuilder
import org.apache.flink.api.table.functions.ScalarFunction
import org.apache.flink.api.table.functions.utils.UserDefinedFunctionUtils.{getResultType, getSignature, signatureToString, signaturesToString}
-import org.apache.flink.api.table.validate.{ExprValidationResult, ValidationFailure, ValidationSuccess}
+import org.apache.flink.api.table.validate.{ValidationResult, ValidationFailure, ValidationSuccess}
import org.apache.flink.api.table.{FlinkTypeFactory, UnresolvedException}
/**
@@ -41,7 +41,7 @@ case class Call(functionName: String, args: Seq[Expression]) extends Expression
override private[flink] def resultType =
throw UnresolvedException(s"calling resultType on UnresolvedFunction $functionName")
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
ValidationFailure(s"Unresolved function call: $functionName")
}
@@ -71,7 +71,7 @@ case class ScalarFunctionCall(
override private[flink] def resultType = getResultType(scalarFunction, foundSignature.get)
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
val signature = children.map(_.resultType)
// look for a signature that matches the input types
foundSignature = getSignature(scalarFunction, signature)
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/cast.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/cast.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/cast.scala
index f65dd5b..2232a91 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/cast.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/cast.scala
@@ -43,7 +43,7 @@ case class Cast(child: Expression, resultType: TypeInformation[_]) extends Unary
copy(child, resultType).asInstanceOf[this.type]
}
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (TypeCoercion.canCast(child.resultType, resultType)) {
ValidationSuccess
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/comparison.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/comparison.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/comparison.scala
index babb677..d5244d0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/comparison.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/comparison.scala
@@ -37,7 +37,7 @@ abstract class BinaryComparison extends BinaryExpression {
override private[flink] def resultType = BOOLEAN_TYPE_INFO
// TODO: tighten this rule once we implemented type coercion rules during validation
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
(left.resultType, right.resultType) match {
case (lType, rType) if isNumeric(lType) && isNumeric(rType) => ValidationSuccess
case (lType, rType) if isComparable(lType) && lType == rType => ValidationSuccess
@@ -53,7 +53,7 @@ case class EqualTo(left: Expression, right: Expression) extends BinaryComparison
private[flink] val sqlOperator: SqlOperator = SqlStdOperatorTable.EQUALS
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
(left.resultType, right.resultType) match {
case (lType, rType) if isNumeric(lType) && isNumeric(rType) => ValidationSuccess
// TODO widen this rule once we support custom objects as types (FLINK-3916)
@@ -68,7 +68,7 @@ case class NotEqualTo(left: Expression, right: Expression) extends BinaryCompari
private[flink] val sqlOperator: SqlOperator = SqlStdOperatorTable.NOT_EQUALS
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
(left.resultType, right.resultType) match {
case (lType, rType) if isNumeric(lType) && isNumeric(rType) => ValidationSuccess
// TODO widen this rule once we support custom objects as types (FLINK-3916)
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/fieldExpression.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/fieldExpression.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/fieldExpression.scala
index 5f20751..91efd08 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/fieldExpression.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/fieldExpression.scala
@@ -19,11 +19,9 @@ package org.apache.flink.api.table.expressions
import org.apache.calcite.rex.RexNode
import org.apache.calcite.tools.RelBuilder
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.table.UnresolvedException
-import org.apache.flink.api.table.validate.{ValidationSuccess, ExprValidationResult,
-ValidationFailure}
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.table.{UnresolvedException, ValidationException}
+import org.apache.flink.api.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
trait NamedExpression extends Expression {
private[flink] def name: String
@@ -44,10 +42,10 @@ case class UnresolvedFieldReference(name: String) extends Attribute {
UnresolvedFieldReference(newName)
override private[flink] def resultType: TypeInformation[_] =
- throw new UnresolvedException(s"calling resultType on ${this.getClass}")
+ throw UnresolvedException(s"Calling resultType on ${this.getClass}.")
- override private[flink] def validateInput(): ExprValidationResult =
- ValidationFailure(s"Unresolved reference $name")
+ override private[flink] def validateInput(): ValidationResult =
+ ValidationFailure(s"Unresolved reference $name.")
}
case class ResolvedFieldReference(
@@ -93,7 +91,7 @@ case class Alias(child: Expression, name: String)
}
}
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (name == "*") {
ValidationFailure("Alias can not accept '*' as name.")
} else {
@@ -115,3 +113,38 @@ case class UnresolvedAlias(child: Expression) extends UnaryExpression with Named
override private[flink] lazy val valid = false
}
+
+case class RowtimeAttribute() extends Attribute {
+ override private[flink] def withName(newName: String): Attribute = {
+ if (newName == "rowtime") {
+ this
+ } else {
+ throw new ValidationException("Cannot rename streaming rowtime attribute.")
+ }
+ }
+
+ override private[flink] def name: String = "rowtime"
+
+ override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
+ throw new UnsupportedOperationException("A rowtime attribute can not be used solely.")
+ }
+
+ override private[flink] def resultType: TypeInformation[_] = BasicTypeInfo.LONG_TYPE_INFO
+}
+
+case class WindowReference(name: String) extends Attribute {
+
+ override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode =
+ throw new UnsupportedOperationException("A window reference can not be used solely.")
+
+ override private[flink] def resultType: TypeInformation[_] =
+ throw new UnsupportedOperationException("A window reference has no result type.")
+
+ override private[flink] def withName(newName: String): Attribute = {
+ if (newName == name) {
+ this
+ } else {
+ throw new ValidationException("Cannot rename window reference.")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/literals.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/literals.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/literals.scala
index 677160a..6382abe 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/literals.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/literals.scala
@@ -28,7 +28,7 @@ import org.apache.calcite.sql.parser.SqlParserPos
import org.apache.calcite.tools.RelBuilder
import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, SqlTimeTypeInfo, TypeInformation}
import org.apache.flink.api.table.FlinkTypeFactory
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo
+import org.apache.flink.api.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo}
object Literal {
private[flink] def apply(l: Any): Literal = l match {
@@ -50,7 +50,16 @@ object Literal {
}
case class Literal(value: Any, resultType: TypeInformation[_]) extends LeafExpression {
- override def toString = s"$value"
+ override def toString = resultType match {
+ case _: BasicTypeInfo[_] => value.toString
+ case _@SqlTimeTypeInfo.DATE => value.toString + ".toDate"
+ case _@SqlTimeTypeInfo.TIME => value.toString + ".toTime"
+ case _@SqlTimeTypeInfo.TIMESTAMP => value.toString + ".toTimestamp"
+ case _@TimeIntervalTypeInfo.INTERVAL_MILLIS => value.toString + ".millis"
+ case _@TimeIntervalTypeInfo.INTERVAL_MONTHS => value.toString + ".months"
+ case _@RowIntervalTypeInfo.INTERVAL_ROWS => value.toString + ".rows"
+ case _ => s"Literal($value, $resultType)"
+ }
override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode = {
resultType match {
@@ -67,7 +76,7 @@ case class Literal(value: Any, resultType: TypeInformation[_]) extends LeafExpre
case SqlTimeTypeInfo.TIMESTAMP =>
relBuilder.getRexBuilder.makeTimestampLiteral(dateToCalendar, 3)
- case IntervalTypeInfo.INTERVAL_MONTHS =>
+ case TimeIntervalTypeInfo.INTERVAL_MONTHS =>
val interval = java.math.BigDecimal.valueOf(value.asInstanceOf[Int])
val intervalQualifier = new SqlIntervalQualifier(
TimeUnit.YEAR,
@@ -75,7 +84,7 @@ case class Literal(value: Any, resultType: TypeInformation[_]) extends LeafExpre
SqlParserPos.ZERO)
relBuilder.getRexBuilder.makeIntervalLiteral(interval, intervalQualifier)
- case IntervalTypeInfo.INTERVAL_MILLIS =>
+ case TimeIntervalTypeInfo.INTERVAL_MILLIS =>
val interval = java.math.BigDecimal.valueOf(value.asInstanceOf[Long])
val intervalQualifier = new SqlIntervalQualifier(
TimeUnit.DAY,
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/logic.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/logic.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/logic.scala
index bf4dc85..9c8e279 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/logic.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/logic.scala
@@ -27,7 +27,7 @@ import org.apache.flink.api.table.validate._
abstract class BinaryPredicate extends BinaryExpression {
override private[flink] def resultType = BasicTypeInfo.BOOLEAN_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (left.resultType == BasicTypeInfo.BOOLEAN_TYPE_INFO &&
right.resultType == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
ValidationSuccess
@@ -48,7 +48,7 @@ case class Not(child: Expression) extends UnaryExpression {
override private[flink] def resultType = BasicTypeInfo.BOOLEAN_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (child.resultType == BasicTypeInfo.BOOLEAN_TYPE_INFO) {
ValidationSuccess
} else {
@@ -94,7 +94,7 @@ case class If(
relBuilder.call(SqlStdOperatorTable.CASE, c, t, f)
}
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (condition.resultType == BasicTypeInfo.BOOLEAN_TYPE_INFO &&
ifTrue.resultType == ifFalse.resultType) {
ValidationSuccess
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/mathExpressions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/mathExpressions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/mathExpressions.scala
index 80adf5f..e0f4691 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/mathExpressions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/mathExpressions.scala
@@ -29,7 +29,7 @@ import org.apache.flink.api.table.validate._
case class Abs(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = child.resultType
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
TypeCheckUtils.assertNumericExpr(child.resultType, "Abs")
override def toString: String = s"abs($child)"
@@ -42,7 +42,7 @@ case class Abs(child: Expression) extends UnaryExpression {
case class Ceil(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = LONG_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
TypeCheckUtils.assertNumericExpr(child.resultType, "Ceil")
override def toString: String = s"ceil($child)"
@@ -68,7 +68,7 @@ case class Exp(child: Expression) extends UnaryExpression with InputTypeSpec {
case class Floor(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = LONG_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult =
+ override private[flink] def validateInput(): ValidationResult =
TypeCheckUtils.assertNumericExpr(child.resultType, "Floor")
override def toString: String = s"floor($child)"
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ordering.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ordering.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ordering.scala
index 85da69c..c15d462 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ordering.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/ordering.scala
@@ -24,7 +24,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.table.validate._
abstract class Ordering extends UnaryExpression {
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!child.isInstanceOf[NamedExpression]) {
ValidationFailure(s"Sort should only based on field reference")
} else {
[5/5] flink git commit: [FLINK-4691] [table] Rework window property
extraction.
Posted by fh...@apache.org.
[FLINK-4691] [table] Rework window property extraction.
- Deduplicate aggregations and window properties in Table API
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/de03e0ce
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/de03e0ce
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/de03e0ce
Branch: refs/heads/master
Commit: de03e0cea16006f04b5f62d3ff70f583f5db9e4f
Parents: 44f3977
Author: Fabian Hueske <fh...@apache.org>
Authored: Wed Oct 26 01:08:43 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Oct 26 23:03:53 2016 +0200
----------------------------------------------------------------------
.../api/table/plan/ProjectionTranslator.scala | 152 +++++++++++++------
.../nodes/datastream/DataStreamAggregate.scala | 82 ++++++++--
.../AggregateAllTimeWindowFunction.scala | 53 +++++++
.../aggregate/AggregateAllWindowFunction.scala | 19 +--
.../aggregate/AggregateTimeWindowFunction.scala | 57 +++++++
.../aggregate/AggregateWindowFunction.scala | 19 +--
.../runtime/aggregate/PropertyCollector.scala | 42 -----
.../aggregate/TimeWindowPropertyCollector.scala | 54 +++++++
.../table/runtime/aggregate/WindowEndRead.scala | 38 -----
.../runtime/aggregate/WindowPropertyRead.scala | 33 ----
.../runtime/aggregate/WindowStartRead.scala | 38 -----
.../org/apache/flink/api/table/table.scala | 40 ++---
.../scala/stream/table/AggregationsITCase.scala | 18 +--
.../scala/stream/table/GroupWindowTest.scala | 48 +++++-
14 files changed, 424 insertions(+), 269 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
index 2299bd1..d09b03e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
@@ -27,78 +27,144 @@ import scala.collection.mutable.ListBuffer
object ProjectionTranslator {
/**
- * Extracts all aggregation and property expressions (zero, one, or more) from an expression,
- * and replaces the original expressions by field accesses expressions.
+ * Extracts and deduplicates all aggregation and window property expressions (zero, one, or more)
+ * from all expressions and replaces the original expressions by field accesses expressions.
+ *
+ * @param exprs a list of expressions to convert
+ * @param tableEnv the TableEnvironment
+ * @return a Tuple3, the first field contains the converted expressions, the second field the
+ * extracted and deduplicated aggregations, and the third field the extracted and
+ * deduplicated window properties.
*/
def extractAggregationsAndProperties(
- exp: Expression,
+ exprs: Seq[Expression],
tableEnv: TableEnvironment)
- : (Expression, List[NamedExpression], List[NamedExpression]) = {
+ : (Seq[NamedExpression], Seq[NamedExpression], Seq[NamedExpression]) = {
+
+ val (aggNames, propNames) =
+ exprs.foldLeft( (Map[Expression, String](), Map[Expression, String]()) ) {
+ (x, y) => identifyAggregationsAndProperties(y, tableEnv, x._1, x._2)
+ }
+
+ val replaced = exprs
+ .map(replaceAggregationsAndProperties(_, tableEnv, aggNames, propNames))
+ .map {
+ case e: Expression => UnresolvedAlias(e)
+ }
+ val aggs = aggNames.map( a => Alias(a._1, a._2)).toSeq
+ val props = propNames.map( p => Alias(p._1, p._2)).toSeq
+
+ (replaced, aggs, props)
+ }
+
+ /** Identifies and deduplicates aggregation functions and window properties. */
+ private def identifyAggregationsAndProperties(
+ exp: Expression,
+ tableEnv: TableEnvironment,
+ aggNames: Map[Expression, String],
+ propNames: Map[Expression, String]) : (Map[Expression, String], Map[Expression, String]) = {
+
+ exp match {
+ case agg: Aggregation =>
+ if (aggNames contains agg) {
+ (aggNames, propNames)
+ } else {
+ (aggNames + (agg -> tableEnv.createUniqueAttributeName()), propNames)
+ }
+ case prop: WindowProperty =>
+ if (propNames contains prop) {
+ (aggNames, propNames)
+ } else {
+ (aggNames, propNames + (prop -> tableEnv.createUniqueAttributeName()))
+ }
+ case l: LeafExpression =>
+ (aggNames, propNames)
+ case u: UnaryExpression =>
+ identifyAggregationsAndProperties(u.child, tableEnv, aggNames, propNames)
+ case b: BinaryExpression =>
+ val l = identifyAggregationsAndProperties(b.left, tableEnv, aggNames, propNames)
+ identifyAggregationsAndProperties(b.right, tableEnv, l._1, l._2)
+
+ // Functions calls
+ case c @ Call(name, args) =>
+ args.foldLeft((aggNames, propNames)){
+ (x, y) => identifyAggregationsAndProperties(y, tableEnv, x._1, x._2)
+ }
+
+ case sfc @ ScalarFunctionCall(clazz, args) =>
+ args.foldLeft((aggNames, propNames)){
+ (x, y) => identifyAggregationsAndProperties(y, tableEnv, x._1, x._2)
+ }
+
+ // General expression
+ case e: Expression =>
+ e.productIterator.foldLeft((aggNames, propNames)){
+ (x, y) => y match {
+ case e: Expression => identifyAggregationsAndProperties(e, tableEnv, x._1, x._2)
+ case _ => (x._1, x._2)
+ }
+ }
+ }
+ }
+
+ /** Replaces aggregations and projections by named field references. */
+ private def replaceAggregationsAndProperties(
+ exp: Expression,
+ tableEnv: TableEnvironment,
+ aggNames: Map[Expression, String],
+ propNames: Map[Expression, String]) : Expression = {
exp match {
case agg: Aggregation =>
- val name = tableEnv.createUniqueAttributeName()
- val aggCall = Alias(agg, name)
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, List(aggCall), Nil)
+ val name = aggNames(agg)
+ Alias(UnresolvedFieldReference(name), tableEnv.createUniqueAttributeName())
case prop: WindowProperty =>
- val name = tableEnv.createUniqueAttributeName()
- val propCall = Alias(prop, name)
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, Nil, List(propCall))
+ val name = propNames(prop)
+ Alias(UnresolvedFieldReference(name), tableEnv.createUniqueAttributeName())
case n @ Alias(agg: Aggregation, name) =>
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, List(n), Nil)
+ val aName = aggNames(agg)
+ Alias(UnresolvedFieldReference(aName), name)
case n @ Alias(prop: WindowProperty, name) =>
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, Nil, List(n))
- case l: LeafExpression =>
- (l, Nil, Nil)
+ val pName = propNames(prop)
+ Alias(UnresolvedFieldReference(pName), name)
+ case l: LeafExpression => l
case u: UnaryExpression =>
- val c = extractAggregationsAndProperties(u.child, tableEnv)
- (u.makeCopy(Array(c._1)), c._2, c._3)
+ val c = replaceAggregationsAndProperties(u.child, tableEnv, aggNames, propNames)
+ u.makeCopy(Array(c))
case b: BinaryExpression =>
- val l = extractAggregationsAndProperties(b.left, tableEnv)
- val r = extractAggregationsAndProperties(b.right, tableEnv)
- (b.makeCopy(Array(l._1, r._1)),
- l._2 ::: r._2,
- l._3 ::: r._3)
+ val l = replaceAggregationsAndProperties(b.left, tableEnv, aggNames, propNames)
+ val r = replaceAggregationsAndProperties(b.right, tableEnv, aggNames, propNames)
+ b.makeCopy(Array(l, r))
// Functions calls
case c @ Call(name, args) =>
- val newArgs = args.map(extractAggregationsAndProperties(_, tableEnv))
- (c.makeCopy((name :: newArgs.map(_._1) :: Nil).toArray),
- newArgs.flatMap(_._2).toList,
- newArgs.flatMap(_._3).toList)
+ val newArgs = args.map(replaceAggregationsAndProperties(_, tableEnv, aggNames, propNames))
+ c.makeCopy(Array(name, newArgs))
case sfc @ ScalarFunctionCall(clazz, args) =>
- val newArgs = args.map(extractAggregationsAndProperties(_, tableEnv))
- (sfc.makeCopy((clazz :: newArgs.map(_._1) :: Nil).toArray),
- newArgs.flatMap(_._2).toList,
- newArgs.flatMap(_._3).toList)
+ val newArgs: Seq[Expression] = args
+ .map(replaceAggregationsAndProperties(_, tableEnv, aggNames, propNames))
+ sfc.makeCopy(Array(clazz,newArgs))
// General expression
case e: Expression =>
val newArgs = e.productIterator.map {
case arg: Expression =>
- extractAggregationsAndProperties(arg, tableEnv)
+ replaceAggregationsAndProperties(arg, tableEnv, aggNames, propNames)
}
- (e.makeCopy(newArgs.map(_._1).toArray),
- newArgs.flatMap(_._2).toList,
- newArgs.flatMap(_._3).toList)
+ e.makeCopy(newArgs.toArray)
}
}
/**
- * Parses all input expressions to [[UnresolvedAlias]].
- * And expands star to parent's full project list.
+ * Expands an UnresolvedFieldReference("*") to parent's full project list.
*/
- def expandProjectList(exprs: Seq[Expression], parent: LogicalNode): Seq[NamedExpression] = {
- val projectList = new ListBuffer[NamedExpression]
+ def expandProjectList(exprs: Seq[Expression], parent: LogicalNode): Seq[Expression] = {
+ val projectList = new ListBuffer[Expression]
exprs.foreach {
case n: UnresolvedFieldReference if n.name == "*" =>
- projectList ++= parent.output.map(UnresolvedAlias(_))
- case e: Expression => projectList += UnresolvedAlias(e)
+ projectList ++= parent.output.map(a => UnresolvedFieldReference(a.name))
+ case e: Expression => projectList += e
}
projectList
}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
index b9b4561..b4ae3ab 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
@@ -22,19 +22,21 @@ import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rel.core.AggregateCall
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.tuple.Tuple
import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
import org.apache.flink.api.table.expressions._
import org.apache.flink.api.table.plan.logical._
import org.apache.flink.api.table.plan.nodes.FlinkAggregate
-import org.apache.flink.api.table.plan.nodes.datastream.DataStreamAggregate.{createKeyedWindowedStream, createNonKeyedWindowedStream, transformToPropertyReads}
+import org.apache.flink.api.table.plan.nodes.datastream.DataStreamAggregate._
import org.apache.flink.api.table.runtime.aggregate.AggregateUtil._
import org.apache.flink.api.table.runtime.aggregate._
import org.apache.flink.api.table.typeutils.TypeCheckUtils.isTimeInterval
import org.apache.flink.api.table.typeutils.{RowIntervalTypeInfo, RowTypeInfo, TimeIntervalTypeInfo, TypeConverter}
-import org.apache.flink.api.table.{FlinkTypeFactory, Row, StreamTableEnvironment}
+import org.apache.flink.api.table.{TableException, FlinkTypeFactory, Row, StreamTableEnvironment}
import org.apache.flink.streaming.api.datastream.{AllWindowedStream, DataStream, KeyedStream, WindowedStream}
+import org.apache.flink.streaming.api.functions.windowing.{WindowFunction, AllWindowFunction}
import org.apache.flink.streaming.api.windowing.assigners._
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
@@ -115,8 +117,6 @@ class DataStreamAggregate(
getRowType,
grouping)
- val propertyReads = transformToPropertyReads(namedProperties.map(_.property))
-
val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(
tableEnv,
// tell the input operator that this operator currently only supports Rows as input
@@ -148,7 +148,8 @@ class DataStreamAggregate(
val aggOpName = s"groupBy: (${groupingToString(inputType, grouping)}), " +
s"window: ($window), " +
s"select: ($aggString)"
- val aggregateFunction = new AggregateWindowFunction(propertyReads, groupReduceFunction)
+ val aggregateFunction =
+ createWindowAggregationFunction(window, namedProperties, groupReduceFunction)
val keyedStream = mappedInput.keyBy(groupingKeys: _*)
@@ -164,7 +165,8 @@ class DataStreamAggregate(
// global / non-keyed aggregation
else {
val aggOpName = s"window: ($window), select: ($aggString)"
- val aggregateFunction = new AggregateAllWindowFunction(propertyReads, groupReduceFunction)
+ val aggregateFunction =
+ createAllWindowAggregationFunction(window, namedProperties, groupReduceFunction)
val windowedStream = createNonKeyedWindowedStream(window, mappedInput)
.asInstanceOf[AllWindowedStream[Row, DataStreamWindow]]
@@ -197,11 +199,69 @@ class DataStreamAggregate(
object DataStreamAggregate {
- private def transformToPropertyReads(namedProperties: Seq[WindowProperty])
- : Array[WindowPropertyRead[_ <: Any]] = namedProperties.map {
- case WindowStart(_) => new WindowStartRead()
- case WindowEnd(_) => new WindowEndRead()
- }.toArray
+ private def createAllWindowAggregationFunction(
+ window: LogicalWindow,
+ properties: Seq[NamedWindowProperty],
+ aggFunction: RichGroupReduceFunction[Row, Row])
+ : AllWindowFunction[Row, Row, DataStreamWindow] = {
+
+ if (isTimeWindow(window)) {
+ val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
+ new AggregateAllTimeWindowFunction(aggFunction, startPos, endPos)
+ .asInstanceOf[AllWindowFunction[Row, Row, DataStreamWindow]]
+ } else {
+ new AggregateAllWindowFunction(aggFunction)
+ }
+
+ }
+
+ private def createWindowAggregationFunction(
+ window: LogicalWindow,
+ properties: Seq[NamedWindowProperty],
+ aggFunction: RichGroupReduceFunction[Row, Row])
+ : WindowFunction[Row, Row, Tuple, DataStreamWindow] = {
+
+ if (isTimeWindow(window)) {
+ val (startPos, endPos) = computeWindowStartEndPropertyPos(properties)
+ new AggregateTimeWindowFunction(aggFunction, startPos, endPos)
+ .asInstanceOf[WindowFunction[Row, Row, Tuple, DataStreamWindow]]
+ } else {
+ new AggregateWindowFunction(aggFunction)
+ }
+
+ }
+
+ private def isTimeWindow(window: LogicalWindow) = {
+ window match {
+ case ProcessingTimeTumblingGroupWindow(_, size) => isTimeInterval(size.resultType)
+ case ProcessingTimeSlidingGroupWindow(_, size, _) => isTimeInterval(size.resultType)
+ case ProcessingTimeSessionGroupWindow(_, _) => true
+ case EventTimeTumblingGroupWindow(_, _, size) => isTimeInterval(size.resultType)
+ case EventTimeSlidingGroupWindow(_, _, size, _) => isTimeInterval(size.resultType)
+ case EventTimeSessionGroupWindow(_, _, _) => true
+ }
+ }
+
+ def computeWindowStartEndPropertyPos(properties: Seq[NamedWindowProperty])
+ : (Option[Int], Option[Int]) = {
+
+ val propPos = properties.foldRight((None: Option[Int], None: Option[Int], 0)) {
+ (p, x) => p match {
+ case NamedWindowProperty(name, prop) =>
+ prop match {
+ case WindowStart(_) if x._1.isDefined =>
+ throw new TableException("Duplicate WindowStart property encountered. This is a bug.")
+ case WindowStart(_) =>
+ (Some(x._3), x._2, x._3 - 1)
+ case WindowEnd(_) if x._2.isDefined =>
+ throw new TableException("Duplicate WindowEnd property encountered. This is a bug.")
+ case WindowEnd(_) =>
+ (x._1, Some(x._3), x._3 - 1)
+ }
+ }
+ }
+ (propPos._1, propPos._2)
+ }
private def createKeyedWindowedStream(groupWindow: LogicalWindow, stream: KeyedStream[Row, Tuple])
: WindowedStream[Row, Tuple, _ <: DataStreamWindow] = groupWindow match {
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllTimeWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllTimeWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllTimeWindowFunction.scala
new file mode 100644
index 0000000..ceadfe7
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllTimeWindowFunction.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
+import org.apache.flink.api.table.Row
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow
+import org.apache.flink.util.Collector
+
+class AggregateAllTimeWindowFunction(
+ groupReduceFunction: RichGroupReduceFunction[Row, Row],
+ windowStartPos: Option[Int],
+ windowEndPos: Option[Int])
+
+ extends RichAllWindowFunction[Row, Row, TimeWindow] {
+
+ private var collector: TimeWindowPropertyCollector = _
+
+ override def open(parameters: Configuration): Unit = {
+ groupReduceFunction.open(parameters)
+ collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
+ }
+
+ override def apply(window: TimeWindow, input: Iterable[Row], out: Collector[Row]): Unit = {
+
+ // set collector and window
+ collector.wrappedCollector = out
+ collector.timeWindow = window
+
+ // call wrapped reduce function with property collector
+ groupReduceFunction.reduce(input, collector)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
index 86f8a20..53ab948 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
@@ -27,27 +27,14 @@ import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction
import org.apache.flink.streaming.api.windowing.windows.Window
import org.apache.flink.util.Collector
-class AggregateAllWindowFunction(
- propertyReads: Array[WindowPropertyRead[_ <: Any]],
- groupReduceFunction: RichGroupReduceFunction[Row, Row])
- extends RichAllWindowFunction[Row, Row, Window] {
-
- private var propertyCollector: PropertyCollector = _
+class AggregateAllWindowFunction(groupReduceFunction: RichGroupReduceFunction[Row, Row])
+ extends RichAllWindowFunction[Row, Row, Window] {
override def open(parameters: Configuration): Unit = {
groupReduceFunction.open(parameters)
- propertyCollector = new PropertyCollector(propertyReads)
}
override def apply(window: Window, input: Iterable[Row], out: Collector[Row]): Unit = {
-
- // extract the properties from window
- propertyReads.foreach(_.extract(window))
-
- // set final collector
- propertyCollector.finalCollector = out
-
- // call wrapped reduce function with property collector
- groupReduceFunction.reduce(input, propertyCollector)
+ groupReduceFunction.reduce(input, out)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateTimeWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateTimeWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateTimeWindowFunction.scala
new file mode 100644
index 0000000..80f52ca
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateTimeWindowFunction.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
+import org.apache.flink.api.java.tuple.Tuple
+import org.apache.flink.api.table.Row
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow
+import org.apache.flink.util.Collector
+
+class AggregateTimeWindowFunction(
+ groupReduceFunction: RichGroupReduceFunction[Row, Row],
+ windowStartPos: Option[Int],
+ windowEndPos: Option[Int])
+ extends RichWindowFunction[Row, Row, Tuple, TimeWindow] {
+
+ private var collector: TimeWindowPropertyCollector = _
+
+ override def open(parameters: Configuration): Unit = {
+ groupReduceFunction.open(parameters)
+ collector = new TimeWindowPropertyCollector(windowStartPos, windowEndPos)
+ }
+
+ override def apply(
+ key: Tuple,
+ window: TimeWindow,
+ input: Iterable[Row],
+ out: Collector[Row]) : Unit = {
+
+ // set collector and window
+ collector.wrappedCollector = out
+ collector.timeWindow = window
+
+ // call wrapped reduce function with property collector
+ groupReduceFunction.reduce(input, collector)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
index c65ac35..180248f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
@@ -28,32 +28,19 @@ import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction
import org.apache.flink.streaming.api.windowing.windows.Window
import org.apache.flink.util.Collector
-class AggregateWindowFunction(
- propertyReads: Array[WindowPropertyRead[_ <: Any]],
- groupReduceFunction: RichGroupReduceFunction[Row, Row])
+class AggregateWindowFunction(groupReduceFunction: RichGroupReduceFunction[Row, Row])
extends RichWindowFunction[Row, Row, Tuple, Window] {
- private var propertyCollector: PropertyCollector = _
-
override def open(parameters: Configuration): Unit = {
groupReduceFunction.open(parameters)
- propertyCollector = new PropertyCollector(propertyReads)
}
override def apply(
key: Tuple,
window: Window,
input: Iterable[Row],
- out: Collector[Row])
- : Unit = {
-
- // extract the properties from window
- propertyReads.foreach(_.extract(window))
-
- // set final collector
- propertyCollector.finalCollector = out
+ out: Collector[Row]) : Unit = {
- // call wrapped reduce function with property collector
- groupReduceFunction.reduce(input, propertyCollector)
+ groupReduceFunction.reduce(input, out)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
deleted file mode 100644
index 763ed0b..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.runtime.aggregate
-
-import org.apache.flink.api.table.Row
-import org.apache.flink.util.Collector
-
-/**
- * Adds properties to the end of a row before it emits it to the final collector.
- * The collector assumes that the row has placeholders at the end that can be filled.
- */
-class PropertyCollector(properties: Array[WindowPropertyRead[_ <: Any]]) extends Collector[Row] {
- var finalCollector: Collector[Row] = _
-
- override def collect(record: Row): Unit = {
- var i: Int = 0
- while (i < properties.length) {
- val idx = record.productArity - properties.length + i
- record.setField(idx, properties(i).get())
- i = i + 1
- }
- finalCollector.collect(record)
- }
-
- override def close(): Unit = finalCollector.close()
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/TimeWindowPropertyCollector.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/TimeWindowPropertyCollector.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/TimeWindowPropertyCollector.scala
new file mode 100644
index 0000000..9f1c23b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/TimeWindowPropertyCollector.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import org.apache.calcite.runtime.SqlFunctions
+import org.apache.flink.api.table.Row
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow
+import org.apache.flink.util.Collector
+
+/**
+ * Adds TimeWindow properties to specified fields of a row before it emits the row to a wrapped
+ * collector.
+ */
+class TimeWindowPropertyCollector(windowStartOffset: Option[Int], windowEndOffset: Option[Int])
+ extends Collector[Row] {
+
+ var wrappedCollector: Collector[Row] = _
+ var timeWindow: TimeWindow = _
+
+ override def collect(record: Row): Unit = {
+
+ val lastFieldPos = record.productArity - 1
+
+ if (windowStartOffset.isDefined) {
+ record.setField(
+ lastFieldPos + windowStartOffset.get,
+ SqlFunctions.internalToTimestamp(timeWindow.getStart))
+ }
+ if (windowEndOffset.isDefined) {
+ record.setField(
+ lastFieldPos + windowEndOffset.get,
+ SqlFunctions.internalToTimestamp(timeWindow.getEnd))
+ }
+ wrappedCollector.collect(record)
+ }
+
+ override def close(): Unit = wrappedCollector.close()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
deleted file mode 100644
index dd12238..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.runtime.aggregate
-
-import java.sql.Timestamp
-
-import org.apache.calcite.runtime.SqlFunctions
-import org.apache.flink.streaming.api.windowing.windows.{TimeWindow, Window}
-
-class WindowEndRead extends WindowPropertyRead[Timestamp] {
-
- private var ts: Timestamp = _
-
- override def extract(window: Window): Unit = window match {
- case timeWindow: TimeWindow =>
- ts = SqlFunctions.internalToTimestamp(timeWindow.getEnd)
- case _ =>
- ts = null
- }
-
- override def get(): Timestamp = ts
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
deleted file mode 100644
index 88ea30d..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.runtime.aggregate
-
-import org.apache.flink.streaming.api.windowing.windows.Window
-
-/**
- * Base class for reading a window property. The property will be extracted once and
- * can be read multiple times.
- */
-trait WindowPropertyRead[T] extends Serializable {
-
- def extract(window: Window): Unit
-
- def get(): T
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
deleted file mode 100644
index d92cb18..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.runtime.aggregate
-
-import java.sql.Timestamp
-
-import org.apache.calcite.runtime.SqlFunctions
-import org.apache.flink.streaming.api.windowing.windows.{TimeWindow, Window}
-
-class WindowStartRead extends WindowPropertyRead[Timestamp] {
-
- private var ts: Timestamp = _
-
- override def extract(window: Window): Unit = window match {
- case timeWindow: TimeWindow =>
- ts = SqlFunctions.internalToTimestamp(timeWindow.getStart)
- case _ =>
- ts = null
- }
-
- override def get(): Timestamp = ts
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
index ebb4dcb..8528c8a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
@@ -76,16 +76,21 @@ class Table(
* }}}
*/
def select(fields: Expression*): Table = {
- val projectionOnAggregates = fields.map(extractAggregationsAndProperties(_, tableEnv))
- val aggregations = projectionOnAggregates.flatMap(_._2)
- val projectList = expandProjectList(projectionOnAggregates.map(_._1), logicalPlan)
- if (aggregations.nonEmpty) {
+
+ val expandedFields = expandProjectList(fields, logicalPlan)
+ val (projection, aggs, props) = extractAggregationsAndProperties(expandedFields, tableEnv)
+
+ if (props.nonEmpty) {
+ throw ValidationException("Window properties can only be used on windowed tables.")
+ }
+
+ if (aggs.nonEmpty) {
new Table(tableEnv,
- Project(projectList,
- Aggregate(Nil, aggregations, logicalPlan).validate(tableEnv)).validate(tableEnv))
+ Project(projection,
+ Aggregate(Nil, aggs, logicalPlan).validate(tableEnv)).validate(tableEnv))
} else {
new Table(tableEnv,
- Project(projectList, logicalPlan).validate(tableEnv))
+ Project(projection, logicalPlan).validate(tableEnv))
}
}
@@ -672,20 +677,18 @@ class GroupedTable(
*/
def select(fields: Expression*): Table = {
- val projectionOnAggsAndProps = fields.map(extractAggregationsAndProperties(_, table.tableEnv))
- val aggregations = projectionOnAggsAndProps.flatMap(_._2)
- val properties = projectionOnAggsAndProps.flatMap(_._3)
+ val (projection, aggs, props) = extractAggregationsAndProperties(fields, table.tableEnv)
- if (properties.nonEmpty) {
+ if (props.nonEmpty) {
throw ValidationException("Window properties can only be used on windowed tables.")
}
val logical =
Project(
- projectionOnAggsAndProps.map(e => UnresolvedAlias(e._1)),
+ projection,
Aggregate(
groupKey,
- aggregations,
+ aggs,
table.logicalPlan
).validate(table.tableEnv)
).validate(table.tableEnv)
@@ -744,20 +747,19 @@ class GroupWindowedTable(
* }}}
*/
def select(fields: Expression*): Table = {
- val projectionOnAggsAndProps = fields.map(extractAggregationsAndProperties(_, table.tableEnv))
- val aggregations = projectionOnAggsAndProps.flatMap(_._2)
- val properties = projectionOnAggsAndProps.flatMap(_._3)
+
+ val (projection, aggs, props) = extractAggregationsAndProperties(fields, table.tableEnv)
val groupWindow = window.toLogicalWindow
val logical =
Project(
- projectionOnAggsAndProps.map(e => UnresolvedAlias(e._1)),
+ projection,
WindowAggregate(
groupKey,
groupWindow,
- properties,
- aggregations,
+ props,
+ aggs,
table.logicalPlan
).validate(table.tableEnv)
).validate(table.tableEnv)
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
index 6bb513e..2ccbb38 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
@@ -160,21 +160,21 @@ class AggregationsITCase extends StreamingMultipleProgramsTestBase {
val windowedTable = table
.groupBy('string)
.window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
- .select('string, 'int.count, 'w.start, 'w.end)
+ .select('string, 'int.count, 'w.start, 'w.end, 'w.start)
val results = windowedTable.toDataStream[Row]
results.addSink(new StreamITCase.StringSink)
env.execute()
val expected = Seq(
- "Hello world,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
- "Hello world,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.015",
- "Hello world,1,1970-01-01 00:00:00.01,1970-01-01 00:00:00.02",
- "Hello world,1,1970-01-01 00:00:00.015,1970-01-01 00:00:00.025",
- "Hello,2,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
- "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
- "Hi,1,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
- "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01")
+ "Hello world,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01,1970-01-01 00:00:00.0",
+ "Hello world,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.015,1970-01-01 00:00:00.005",
+ "Hello world,1,1970-01-01 00:00:00.01,1970-01-01 00:00:00.02,1970-01-01 00:00:00.01",
+ "Hello world,1,1970-01-01 00:00:00.015,1970-01-01 00:00:00.025,1970-01-01 00:00:00.015",
+ "Hello,2,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005,1969-12-31 23:59:59.995",
+ "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01,1970-01-01 00:00:00.0",
+ "Hi,1,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005,1969-12-31 23:59:59.995",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01,1970-01-01 00:00:00.0")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/de03e0ce/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
index 96fd787..b59b151 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/GroupWindowTest.scala
@@ -586,7 +586,7 @@ class GroupWindowTest extends TableTestBase {
val windowedTable = table
.groupBy('string)
.window(Session withGap 3.milli on 'rowtime as 'w)
- .select('w.end, 'string, 'int.count, 'w.start, 'w.end)
+ .select('w.end as 'we1, 'string, 'int.count as 'cnt, 'w.start as 'ws, 'w.end as 'we2)
val expected = unaryNode(
"DataStreamCalc",
@@ -603,10 +603,50 @@ class GroupWindowTest extends TableTestBase {
"string",
"COUNT(int) AS TMP_1",
"end(WindowReference(w)) AS TMP_0",
- "start(WindowReference(w)) AS TMP_2",
- "end(WindowReference(w)) AS TMP_3")
+ "start(WindowReference(w)) AS TMP_2")
),
- term("select", "TMP_0", "string", "TMP_1", "TMP_2", "TMP_3")
+ term("select", "TMP_0 AS we1", "string", "TMP_1 AS cnt", "TMP_2 AS ws", "TMP_0 AS we2")
+ )
+
+ util.verifyTable(windowedTable, expected)
+ }
+
+ @Test
+ def testTumbleWindowWithDuplicateAggsAndProps(): Unit = {
+ val util = streamTestUtil()
+ val table = util.addTable[(Long, Int, String)]('long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 5.millis on 'rowtime as 'w)
+ .select('string, 'int.sum + 1 as 's1, 'int.sum + 3 as 's2, 'w.start as 'x, 'w.start as 'x2,
+ 'w.end as 'x3, 'w.end)
+
+ val expected = unaryNode(
+ "DataStreamCalc",
+ unaryNode(
+ "DataStreamAggregate",
+ streamTableNode(0),
+ term("groupBy", "string"),
+ term("window",
+ EventTimeTumblingGroupWindow(
+ Some(WindowReference("w")),
+ RowtimeAttribute(),
+ 5.millis)),
+ term("select",
+ "string",
+ "SUM(int) AS TMP_0",
+ "start(WindowReference(w)) AS TMP_1",
+ "end(WindowReference(w)) AS TMP_2")
+ ),
+ term("select",
+ "string",
+ "+(CAST(AS(TMP_0, 'TMP_3')), CAST(1)) AS s1",
+ "+(CAST(AS(TMP_0, 'TMP_4')), CAST(3)) AS s2",
+ "TMP_1 AS x",
+ "TMP_1 AS x2",
+ "TMP_2 AS x3",
+ "TMP_2 AS TMP_5")
)
util.verifyTable(windowedTable, expected)
[3/5] flink git commit: [FLINK-4691] [table] Add group-windows for
streaming tables to Table API.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/stringExpressions.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/stringExpressions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/stringExpressions.scala
index a692c9e..56b5b5e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/stringExpressions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/stringExpressions.scala
@@ -32,7 +32,7 @@ import org.apache.flink.api.table.validate._
case class CharLength(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = INT_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (child.resultType == STRING_TYPE_INFO) {
ValidationSuccess
} else {
@@ -55,7 +55,7 @@ case class CharLength(child: Expression) extends UnaryExpression {
case class InitCap(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (child.resultType == STRING_TYPE_INFO) {
ValidationSuccess
} else {
@@ -80,7 +80,7 @@ case class Like(str: Expression, pattern: Expression) extends BinaryExpression {
override private[flink] def resultType: TypeInformation[_] = BOOLEAN_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (str.resultType == STRING_TYPE_INFO && pattern.resultType == STRING_TYPE_INFO) {
ValidationSuccess
} else {
@@ -102,7 +102,7 @@ case class Like(str: Expression, pattern: Expression) extends BinaryExpression {
case class Lower(child: Expression) extends UnaryExpression {
override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (child.resultType == STRING_TYPE_INFO) {
ValidationSuccess
} else {
@@ -127,7 +127,7 @@ case class Similar(str: Expression, pattern: Expression) extends BinaryExpressio
override private[flink] def resultType: TypeInformation[_] = BOOLEAN_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (str.resultType == STRING_TYPE_INFO && pattern.resultType == STRING_TYPE_INFO) {
ValidationSuccess
} else {
@@ -179,7 +179,7 @@ case class Trim(
override private[flink] def resultType: TypeInformation[_] = STRING_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
trimMode match {
case SymbolExpression(_: TrimMode) =>
if (trimString.resultType != STRING_TYPE_INFO) {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/time.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/time.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/time.scala
index 488fd33..cd5ca0a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/time.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/time.scala
@@ -29,8 +29,8 @@ import org.apache.flink.api.table.FlinkRelBuilder
import org.apache.flink.api.table.expressions.ExpressionUtils.{divide, getFactor, mod}
import org.apache.flink.api.table.expressions.TimeIntervalUnit.TimeIntervalUnit
import org.apache.flink.api.table.typeutils.TypeCheckUtils.isTimeInterval
-import org.apache.flink.api.table.typeutils.{IntervalTypeInfo, TypeCheckUtils}
-import org.apache.flink.api.table.validate.{ExprValidationResult, ValidationFailure, ValidationSuccess}
+import org.apache.flink.api.table.typeutils.{TimeIntervalTypeInfo, TypeCheckUtils}
+import org.apache.flink.api.table.validate.{ValidationResult, ValidationFailure, ValidationSuccess}
import scala.collection.JavaConversions._
@@ -40,7 +40,7 @@ case class Extract(timeIntervalUnit: Expression, temporal: Expression) extends E
override private[flink] def resultType: TypeInformation[_] = LONG_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!TypeCheckUtils.isTemporal(temporal.resultType)) {
return ValidationFailure(s"Extract operator requires Temporal input, " +
s"but $temporal is of type ${temporal.resultType}")
@@ -52,8 +52,8 @@ case class Extract(timeIntervalUnit: Expression, temporal: Expression) extends E
| SymbolExpression(TimeIntervalUnit.DAY)
if temporal.resultType == SqlTimeTypeInfo.DATE
|| temporal.resultType == SqlTimeTypeInfo.TIMESTAMP
- || temporal.resultType == IntervalTypeInfo.INTERVAL_MILLIS
- || temporal.resultType == IntervalTypeInfo.INTERVAL_MONTHS =>
+ || temporal.resultType == TimeIntervalTypeInfo.INTERVAL_MILLIS
+ || temporal.resultType == TimeIntervalTypeInfo.INTERVAL_MONTHS =>
ValidationSuccess
case SymbolExpression(TimeIntervalUnit.HOUR)
@@ -61,7 +61,7 @@ case class Extract(timeIntervalUnit: Expression, temporal: Expression) extends E
| SymbolExpression(TimeIntervalUnit.SECOND)
if temporal.resultType == SqlTimeTypeInfo.TIME
|| temporal.resultType == SqlTimeTypeInfo.TIMESTAMP
- || temporal.resultType == IntervalTypeInfo.INTERVAL_MILLIS =>
+ || temporal.resultType == TimeIntervalTypeInfo.INTERVAL_MILLIS =>
ValidationSuccess
case _ =>
@@ -146,7 +146,7 @@ abstract class TemporalCeilFloor(
override private[flink] def resultType: TypeInformation[_] = temporal.resultType
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!TypeCheckUtils.isTimePoint(temporal.resultType)) {
return ValidationFailure(s"Temporal ceil/floor operator requires Time Point input, " +
s"but $temporal is of type ${temporal.resultType}")
@@ -211,7 +211,7 @@ abstract class CurrentTimePoint(
override private[flink] def resultType: TypeInformation[_] = targetType
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!TypeCheckUtils.isTimePoint(targetType)) {
ValidationFailure(s"CurrentTimePoint operator requires Time Point target type, " +
s"but get $targetType.")
@@ -293,7 +293,7 @@ case class TemporalOverlaps(
override private[flink] def resultType: TypeInformation[_] = BOOLEAN_TYPE_INFO
- override private[flink] def validateInput(): ExprValidationResult = {
+ override private[flink] def validateInput(): ValidationResult = {
if (!TypeCheckUtils.isTimePoint(leftTimePoint.resultType)) {
return ValidationFailure(s"TemporalOverlaps operator requires leftTimePoint to be of type " +
s"Time Point, but get ${leftTimePoint.resultType}.")
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/windowProperties.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/windowProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/windowProperties.scala
new file mode 100644
index 0000000..8386c46
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/expressions/windowProperties.scala
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.expressions
+
+import org.apache.calcite.rex.RexNode
+import org.apache.calcite.tools.RelBuilder
+import org.apache.flink.api.common.typeinfo.SqlTimeTypeInfo
+import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
+import org.apache.flink.api.table.validate.{ValidationFailure, ValidationSuccess}
+
+abstract class WindowProperty(child: Expression) extends UnaryExpression {
+
+ override def toString = s"WindowProperty($child)"
+
+ override private[flink] def toRexNode(implicit relBuilder: RelBuilder): RexNode =
+ throw new UnsupportedOperationException("WindowProperty cannot be transformed to RexNode.")
+
+ override private[flink] def validateInput() =
+ if (child.isInstanceOf[WindowReference]) {
+ ValidationSuccess
+ } else {
+ ValidationFailure("Child must be a window reference.")
+ }
+
+ private[flink] def toNamedWindowProperty(name: String)(implicit relBuilder: RelBuilder)
+ : NamedWindowProperty = NamedWindowProperty(name, this)
+}
+
+case class WindowStart(child: Expression) extends WindowProperty(child) {
+
+ override private[flink] def resultType = SqlTimeTypeInfo.TIMESTAMP
+
+ override def toString: String = s"start($child)"
+}
+
+case class WindowEnd(child: Expression) extends WindowProperty(child) {
+
+ override private[flink] def resultType = SqlTimeTypeInfo.TIMESTAMP
+
+ override def toString: String = s"end($child)"
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
new file mode 100644
index 0000000..2299bd1
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/ProjectionTranslator.scala
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan
+
+import org.apache.flink.api.table.TableEnvironment
+import org.apache.flink.api.table.expressions._
+import org.apache.flink.api.table.plan.logical.LogicalNode
+
+import scala.collection.mutable.ListBuffer
+
+object ProjectionTranslator {
+
+ /**
+ * Extracts all aggregation and property expressions (zero, one, or more) from an expression,
+ * and replaces the original expressions by field accesses expressions.
+ */
+ def extractAggregationsAndProperties(
+ exp: Expression,
+ tableEnv: TableEnvironment)
+ : (Expression, List[NamedExpression], List[NamedExpression]) = {
+
+ exp match {
+ case agg: Aggregation =>
+ val name = tableEnv.createUniqueAttributeName()
+ val aggCall = Alias(agg, name)
+ val fieldExp = UnresolvedFieldReference(name)
+ (fieldExp, List(aggCall), Nil)
+ case prop: WindowProperty =>
+ val name = tableEnv.createUniqueAttributeName()
+ val propCall = Alias(prop, name)
+ val fieldExp = UnresolvedFieldReference(name)
+ (fieldExp, Nil, List(propCall))
+ case n @ Alias(agg: Aggregation, name) =>
+ val fieldExp = UnresolvedFieldReference(name)
+ (fieldExp, List(n), Nil)
+ case n @ Alias(prop: WindowProperty, name) =>
+ val fieldExp = UnresolvedFieldReference(name)
+ (fieldExp, Nil, List(n))
+ case l: LeafExpression =>
+ (l, Nil, Nil)
+ case u: UnaryExpression =>
+ val c = extractAggregationsAndProperties(u.child, tableEnv)
+ (u.makeCopy(Array(c._1)), c._2, c._3)
+ case b: BinaryExpression =>
+ val l = extractAggregationsAndProperties(b.left, tableEnv)
+ val r = extractAggregationsAndProperties(b.right, tableEnv)
+ (b.makeCopy(Array(l._1, r._1)),
+ l._2 ::: r._2,
+ l._3 ::: r._3)
+
+ // Functions calls
+ case c @ Call(name, args) =>
+ val newArgs = args.map(extractAggregationsAndProperties(_, tableEnv))
+ (c.makeCopy((name :: newArgs.map(_._1) :: Nil).toArray),
+ newArgs.flatMap(_._2).toList,
+ newArgs.flatMap(_._3).toList)
+
+ case sfc @ ScalarFunctionCall(clazz, args) =>
+ val newArgs = args.map(extractAggregationsAndProperties(_, tableEnv))
+ (sfc.makeCopy((clazz :: newArgs.map(_._1) :: Nil).toArray),
+ newArgs.flatMap(_._2).toList,
+ newArgs.flatMap(_._3).toList)
+
+ // General expression
+ case e: Expression =>
+ val newArgs = e.productIterator.map {
+ case arg: Expression =>
+ extractAggregationsAndProperties(arg, tableEnv)
+ }
+ (e.makeCopy(newArgs.map(_._1).toArray),
+ newArgs.flatMap(_._2).toList,
+ newArgs.flatMap(_._3).toList)
+ }
+ }
+
+ /**
+ * Parses all input expressions to [[UnresolvedAlias]].
+ * And expands star to parent's full project list.
+ */
+ def expandProjectList(exprs: Seq[Expression], parent: LogicalNode): Seq[NamedExpression] = {
+ val projectList = new ListBuffer[NamedExpression]
+ exprs.foreach {
+ case n: UnresolvedFieldReference if n.name == "*" =>
+ projectList ++= parent.output.map(UnresolvedAlias(_))
+ case e: Expression => projectList += UnresolvedAlias(e)
+ }
+ projectList
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
deleted file mode 100644
index eb40bba..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/RexNodeTranslator.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.plan
-
-import org.apache.flink.api.table.TableEnvironment
-import org.apache.flink.api.table.expressions._
-import org.apache.flink.api.table.plan.logical.LogicalNode
-
-import scala.collection.mutable.ListBuffer
-
-object RexNodeTranslator {
-
- /**
- * Extracts all aggregation expressions (zero, one, or more) from an expression,
- * and replaces the original aggregation expressions by field accesses expressions.
- */
- def extractAggregations(
- exp: Expression,
- tableEnv: TableEnvironment): Pair[Expression, List[NamedExpression]] = {
-
- exp match {
- case agg: Aggregation =>
- val name = tableEnv.createUniqueAttributeName()
- val aggCall = Alias(agg, name)
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, List(aggCall))
- case n @ Alias(agg: Aggregation, name) =>
- val fieldExp = UnresolvedFieldReference(name)
- (fieldExp, List(n))
- case l: LeafExpression =>
- (l, Nil)
- case u: UnaryExpression =>
- val c = extractAggregations(u.child, tableEnv)
- (u.makeCopy(Array(c._1)), c._2)
- case b: BinaryExpression =>
- val l = extractAggregations(b.left, tableEnv)
- val r = extractAggregations(b.right, tableEnv)
- (b.makeCopy(Array(l._1, r._1)), l._2 ::: r._2)
-
- // Functions calls
- case c @ Call(name, args) =>
- val newArgs = args.map(extractAggregations(_, tableEnv))
- (c.makeCopy((name :: newArgs.map(_._1) :: Nil).toArray), newArgs.flatMap(_._2).toList)
-
- case sfc @ ScalarFunctionCall(clazz, args) =>
- val newArgs = args.map(extractAggregations(_, tableEnv))
- (sfc.makeCopy((clazz :: newArgs.map(_._1) :: Nil).toArray), newArgs.flatMap(_._2).toList)
-
- // General expression
- case e: Expression =>
- val newArgs = e.productIterator.map {
- case arg: Expression =>
- extractAggregations(arg, tableEnv)
- }
- (e.makeCopy(newArgs.map(_._1).toArray), newArgs.flatMap(_._2).toList)
- }
- }
-
- /**
- * Parses all input expressions to [[UnresolvedAlias]].
- * And expands star to parent's full project list.
- */
- def expandProjectList(exprs: Seq[Expression], parent: LogicalNode): Seq[NamedExpression] = {
- val projectList = new ListBuffer[NamedExpression]
- exprs.foreach {
- case n: UnresolvedFieldReference if n.name == "*" =>
- projectList ++= parent.output.map(UnresolvedAlias(_))
- case e: Expression => projectList += UnresolvedAlias(e)
- }
- projectList
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalNode.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalNode.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalNode.scala
index dae02bd..55fba07 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalNode.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalNode.scala
@@ -19,8 +19,7 @@ package org.apache.flink.api.table.plan.logical
import org.apache.calcite.rel.RelNode
import org.apache.calcite.tools.RelBuilder
-
-import org.apache.flink.api.table.{TableEnvironment, ValidationException}
+import org.apache.flink.api.table.{StreamTableEnvironment, TableEnvironment, ValidationException}
import org.apache.flink.api.table.expressions._
import org.apache.flink.api.table.trees.TreeNode
import org.apache.flink.api.table.typeutils.TypeCoercion
@@ -54,7 +53,7 @@ abstract class LogicalNode extends TreeNode[LogicalNode] {
// resolve references and function calls
val exprResolved = expressionPostOrderTransform {
case u @ UnresolvedFieldReference(name) =>
- resolveReference(name).getOrElse(u)
+ resolveReference(tableEnv, name).getOrElse(u)
case c @ Call(name, children) if c.childrenValid =>
tableEnv.getFunctionCatalog.lookupFunction(name, children)
}
@@ -84,7 +83,7 @@ abstract class LogicalNode extends TreeNode[LogicalNode] {
resolvedNode.expressionPostOrderTransform {
case a: Attribute if !a.valid =>
val from = children.flatMap(_.output).map(_.name).mkString(", ")
- failValidation(s"cannot resolve [${a.name}] given input [$from]")
+ failValidation(s"Cannot resolve [${a.name}] given input [$from].")
case e: Expression if e.validateInput().isFailure =>
failValidation(s"Expression $e failed on input check: " +
@@ -96,12 +95,12 @@ abstract class LogicalNode extends TreeNode[LogicalNode] {
* Resolves the given strings to a [[NamedExpression]] using the input from all child
* nodes of this LogicalPlan.
*/
- def resolveReference(name: String): Option[NamedExpression] = {
+ def resolveReference(tableEnv: TableEnvironment, name: String): Option[NamedExpression] = {
val childrenOutput = children.flatMap(_.output)
val candidates = childrenOutput.filter(_.name.equalsIgnoreCase(name))
if (candidates.length > 1) {
- failValidation(s"Reference $name is ambiguous")
- } else if (candidates.length == 0) {
+ failValidation(s"Reference $name is ambiguous.")
+ } else if (candidates.isEmpty) {
None
} else {
Some(candidates.head.withName(name))
@@ -133,6 +132,7 @@ abstract class LogicalNode extends TreeNode[LogicalNode] {
case e: Expression => expressionPostOrderTransform(e)
case other => other
}
+ case r: Resolvable[_] => r.resolveExpressions(e => expressionPostOrderTransform(e))
case other: AnyRef => other
}.toArray
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalWindow.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalWindow.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalWindow.scala
new file mode 100644
index 0000000..19fd603
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/LogicalWindow.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.logical
+
+import org.apache.flink.api.table.TableEnvironment
+import org.apache.flink.api.table.expressions.{Expression, WindowReference}
+import org.apache.flink.api.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
+
+abstract class LogicalWindow(val alias: Option[Expression]) extends Resolvable[LogicalWindow] {
+
+ def resolveExpressions(resolver: (Expression) => Expression): LogicalWindow = this
+
+ def validate(tableEnv: TableEnvironment): ValidationResult = alias match {
+ case Some(WindowReference(_)) => ValidationSuccess
+ case Some(_) => ValidationFailure("Window reference for window expected.")
+ case None => ValidationSuccess
+ }
+
+ override def toString: String = getClass.getSimpleName
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/Resolvable.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/Resolvable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/Resolvable.scala
new file mode 100644
index 0000000..7540d43
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/Resolvable.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.logical
+
+import org.apache.flink.api.table.expressions.Expression
+
+/**
+ * A class implementing this interface can resolve the expressions of its parameters and
+ * return a new instance with resolved parameters. This is necessary if expression are nested in
+ * a not supported structure. By default, the validation of a logical node can resolve common
+ * structures like `Expression`, `Option[Expression]`, `Traversable[Expression]`.
+ *
+ * See also [[LogicalNode.expressionPostOrderTransform(scala.PartialFunction)]].
+ *
+ * @tparam T class which expression parameters need to be resolved
+ */
+trait Resolvable[T <: AnyRef] {
+
+ /**
+ * An implementing class can resolve its expressions by applying the given resolver
+ * function on its parameters.
+ *
+ * @param resolver function that can resolve an expression
+ * @return class with resolved expression parameters
+ */
+ def resolveExpressions(resolver: (Expression) => Expression): T
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/groupWindows.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/groupWindows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/groupWindows.scala
new file mode 100644
index 0000000..aeb9676
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/groupWindows.scala
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.logical
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.table.{BatchTableEnvironment, StreamTableEnvironment, TableEnvironment}
+import org.apache.flink.api.table.expressions._
+import org.apache.flink.api.table.typeutils.{RowIntervalTypeInfo, TimeIntervalTypeInfo, TypeCoercion}
+import org.apache.flink.api.table.validate.{ValidationFailure, ValidationResult, ValidationSuccess}
+
+abstract class EventTimeGroupWindow(
+ name: Option[Expression],
+ time: Expression)
+ extends LogicalWindow(name) {
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult = {
+ val valid = super.validate(tableEnv)
+ if (valid.isFailure) {
+ return valid
+ }
+
+ tableEnv match {
+ case _: StreamTableEnvironment =>
+ time match {
+ case RowtimeAttribute() =>
+ ValidationSuccess
+ case _ =>
+ ValidationFailure("Event-time window expects a 'rowtime' time field.")
+ }
+ case _: BatchTableEnvironment =>
+ if (!TypeCoercion.canCast(time.resultType, BasicTypeInfo.LONG_TYPE_INFO)) {
+ ValidationFailure(s"Event-time window expects a time field that can be safely cast " +
+ s"to Long, but is ${time.resultType}")
+ } else {
+ ValidationSuccess
+ }
+ }
+
+ }
+}
+
+abstract class ProcessingTimeGroupWindow(name: Option[Expression]) extends LogicalWindow(name)
+
+// ------------------------------------------------------------------------------------------------
+// Tumbling group windows
+// ------------------------------------------------------------------------------------------------
+
+object TumblingGroupWindow {
+ def validate(tableEnv: TableEnvironment, size: Expression): ValidationResult = size match {
+ case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+ ValidationSuccess
+ case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
+ ValidationSuccess
+ case _ =>
+ ValidationFailure("Tumbling window expects size literal of type Interval of Milliseconds " +
+ "or Interval of Rows.")
+ }
+}
+
+case class ProcessingTimeTumblingGroupWindow(
+ name: Option[Expression],
+ size: Expression)
+ extends ProcessingTimeGroupWindow(name) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ ProcessingTimeTumblingGroupWindow(
+ name.map(resolve),
+ resolve(size))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv).orElse(TumblingGroupWindow.validate(tableEnv, size))
+
+ override def toString: String = s"ProcessingTimeTumblingGroupWindow($name, $size)"
+}
+
+case class EventTimeTumblingGroupWindow(
+ name: Option[Expression],
+ timeField: Expression,
+ size: Expression)
+ extends EventTimeGroupWindow(
+ name,
+ timeField) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ EventTimeTumblingGroupWindow(
+ name.map(resolve),
+ resolve(timeField),
+ resolve(size))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv)
+ .orElse(TumblingGroupWindow.validate(tableEnv, size))
+ .orElse(size match {
+ case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
+ ValidationFailure(
+ "Event-time grouping windows on row intervals are currently not supported.")
+ case _ =>
+ ValidationSuccess
+ })
+
+ override def toString: String = s"EventTimeTumblingGroupWindow($name, $timeField, $size)"
+}
+
+// ------------------------------------------------------------------------------------------------
+// Sliding group windows
+// ------------------------------------------------------------------------------------------------
+
+object SlidingGroupWindow {
+ def validate(
+ tableEnv: TableEnvironment,
+ size: Expression,
+ slide: Expression)
+ : ValidationResult = {
+
+ val checkedSize = size match {
+ case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+ ValidationSuccess
+ case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
+ ValidationSuccess
+ case _ =>
+ ValidationFailure("Sliding window expects size literal of type Interval of " +
+ "Milliseconds or Interval of Rows.")
+ }
+
+ val checkedSlide = slide match {
+ case Literal(_, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+ ValidationSuccess
+ case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
+ ValidationSuccess
+ case _ =>
+ ValidationFailure("Sliding window expects slide literal of type Interval of " +
+ "Milliseconds or Interval of Rows.")
+ }
+
+ checkedSize
+ .orElse(checkedSlide)
+ .orElse {
+ if (size.resultType != slide.resultType) {
+ ValidationFailure("Sliding window expects same type of size and slide.")
+ } else {
+ ValidationSuccess
+ }
+ }
+ }
+}
+
+case class ProcessingTimeSlidingGroupWindow(
+ name: Option[Expression],
+ size: Expression,
+ slide: Expression)
+ extends ProcessingTimeGroupWindow(name) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ ProcessingTimeSlidingGroupWindow(
+ name.map(resolve),
+ resolve(size),
+ resolve(slide))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv).orElse(SlidingGroupWindow.validate(tableEnv, size, slide))
+
+ override def toString: String = s"ProcessingTimeSlidingGroupWindow($name, $size, $slide)"
+}
+
+case class EventTimeSlidingGroupWindow(
+ name: Option[Expression],
+ timeField: Expression,
+ size: Expression,
+ slide: Expression)
+ extends EventTimeGroupWindow(name, timeField) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ EventTimeSlidingGroupWindow(
+ name.map(resolve),
+ resolve(timeField),
+ resolve(size),
+ resolve(slide))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv)
+ .orElse(SlidingGroupWindow.validate(tableEnv, size, slide))
+ .orElse(size match {
+ case Literal(_, RowIntervalTypeInfo.INTERVAL_ROWS) =>
+ ValidationFailure(
+ "Event-time grouping windows on row intervals are currently not supported.")
+ case _ =>
+ ValidationSuccess
+ })
+
+ override def toString: String = s"EventTimeSlidingGroupWindow($name, $timeField, $size, $slide)"
+}
+
+// ------------------------------------------------------------------------------------------------
+// Session group windows
+// ------------------------------------------------------------------------------------------------
+
+object SessionGroupWindow {
+
+ def validate(tableEnv: TableEnvironment, gap: Expression): ValidationResult = gap match {
+ case Literal(timeInterval: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) =>
+ ValidationSuccess
+ case _ =>
+ ValidationFailure(
+ "Session window expects gap literal of type Interval of Milliseconds.")
+ }
+}
+
+case class ProcessingTimeSessionGroupWindow(
+ name: Option[Expression],
+ gap: Expression)
+ extends ProcessingTimeGroupWindow(name) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ ProcessingTimeSessionGroupWindow(
+ name.map(resolve),
+ resolve(gap))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv).orElse(SessionGroupWindow.validate(tableEnv, gap))
+
+ override def toString: String = s"ProcessingTimeSessionGroupWindow($name, $gap)"
+}
+
+case class EventTimeSessionGroupWindow(
+ name: Option[Expression],
+ timeField: Expression,
+ gap: Expression)
+ extends EventTimeGroupWindow(
+ name,
+ timeField) {
+
+ override def resolveExpressions(resolve: (Expression) => Expression): LogicalWindow =
+ EventTimeSessionGroupWindow(
+ name.map(resolve),
+ resolve(timeField),
+ resolve(gap))
+
+ override def validate(tableEnv: TableEnvironment): ValidationResult =
+ super.validate(tableEnv).orElse(SessionGroupWindow.validate(tableEnv, gap))
+
+ override def toString: String = s"EventTimeSessionGroupWindow($name, $timeField, $gap)"
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/operators.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/operators.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/operators.scala
index 066e9d6..1d7ed5f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/operators.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/operators.scala
@@ -28,6 +28,7 @@ import org.apache.flink.api.java.operators.join.JoinType
import org.apache.flink.api.table._
import org.apache.flink.api.table.expressions._
import org.apache.flink.api.table.typeutils.TypeConverter
+import org.apache.flink.api.table.validate.{ValidationFailure, ValidationSuccess}
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -55,28 +56,27 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalNode) extend
override def validate(tableEnv: TableEnvironment): LogicalNode = {
val resolvedProject = super.validate(tableEnv).asInstanceOf[Project]
+ val names: mutable.Set[String] = mutable.Set()
- def checkUniqueNames(exprs: Seq[Expression]): Unit = {
- val names: mutable.Set[String] = mutable.Set()
- exprs.foreach {
- case n: Alias =>
- // explicit name
- if (names.contains(n.name)) {
- throw ValidationException(s"Duplicate field name ${n.name}.")
- } else {
- names.add(n.name)
- }
- case r: ResolvedFieldReference =>
- // simple field forwarding
- if (names.contains(r.name)) {
- throw ValidationException(s"Duplicate field name ${r.name}.")
- } else {
- names.add(r.name)
- }
- case _ => // Do nothing
+ def checkName(name: String): Unit = {
+ if (names.contains(name)) {
+ failValidation(s"Duplicate field name $name.")
+ } else if (tableEnv.isInstanceOf[StreamTableEnvironment] && name == "rowtime") {
+ failValidation("'rowtime' cannot be used as field name in a streaming environment.")
+ } else {
+ names.add(name)
}
}
- checkUniqueNames(resolvedProject.projectList)
+
+ resolvedProject.projectList.foreach {
+ case n: Alias =>
+ // explicit name
+ checkName(n.name)
+ case r: ResolvedFieldReference =>
+ // simple field forwarding
+ checkName(r.name)
+ case _ => // Do nothing
+ }
resolvedProject
}
@@ -112,6 +112,10 @@ case class AliasNode(aliasList: Seq[Expression], child: LogicalNode) extends Una
failValidation("Alias only accept name expressions as arguments")
} else if (!aliasList.forall(_.asInstanceOf[UnresolvedFieldReference].name != "*")) {
failValidation("Alias can not accept '*' as name")
+ } else if (tableEnv.isInstanceOf[StreamTableEnvironment] && !aliasList.forall {
+ case UnresolvedFieldReference(name) => name != "rowtime"
+ }) {
+ failValidation("'rowtime' cannot be used as field name in a streaming environment.")
} else {
val names = aliasList.map(_.asInstanceOf[UnresolvedFieldReference].name)
val input = child.output
@@ -498,3 +502,105 @@ case class LogicalRelNode(
override def validate(tableEnv: TableEnvironment): LogicalNode = this
}
+
+case class WindowAggregate(
+ groupingExpressions: Seq[Expression],
+ window: LogicalWindow,
+ propertyExpressions: Seq[NamedExpression],
+ aggregateExpressions: Seq[NamedExpression],
+ child: LogicalNode)
+ extends UnaryNode {
+
+ override def output: Seq[Attribute] = {
+ (groupingExpressions ++ aggregateExpressions ++ propertyExpressions) map {
+ case ne: NamedExpression => ne.toAttribute
+ case e => Alias(e, e.toString).toAttribute
+ }
+ }
+
+ // resolve references of this operator's parameters
+ override def resolveReference(
+ tableEnv: TableEnvironment,
+ name: String)
+ : Option[NamedExpression] = tableEnv match {
+ // resolve reference to rowtime attribute in a streaming environment
+ case _: StreamTableEnvironment if name == "rowtime" =>
+ Some(RowtimeAttribute())
+ case _ =>
+ window.alias match {
+ // resolve reference to this window's alias
+ case Some(UnresolvedFieldReference(alias)) if name == alias =>
+ // check if reference can already be resolved by input fields
+ val found = super.resolveReference(tableEnv, name)
+ if (found.isDefined) {
+ failValidation(s"Reference $name is ambiguous.")
+ } else {
+ Some(WindowReference(name))
+ }
+ case _ =>
+ // resolve references as usual
+ super.resolveReference(tableEnv, name)
+ }
+ }
+
+ override protected[logical] def construct(relBuilder: RelBuilder): RelBuilder = {
+ val flinkRelBuilder = relBuilder.asInstanceOf[FlinkRelBuilder]
+ child.construct(flinkRelBuilder)
+ flinkRelBuilder.aggregate(
+ window,
+ relBuilder.groupKey(groupingExpressions.map(_.toRexNode(relBuilder)).asJava),
+ propertyExpressions.map {
+ case Alias(prop: WindowProperty, name) => prop.toNamedWindowProperty(name)(relBuilder)
+ case _ => throw new RuntimeException("This should never happen.")
+ },
+ aggregateExpressions.map {
+ case Alias(agg: Aggregation, name) => agg.toAggCall(name)(relBuilder)
+ case _ => throw new RuntimeException("This should never happen.")
+ }.asJava)
+ }
+
+ override def validate(tableEnv: TableEnvironment): LogicalNode = {
+ val resolvedWindowAggregate = super.validate(tableEnv).asInstanceOf[WindowAggregate]
+ val groupingExprs = resolvedWindowAggregate.groupingExpressions
+ val aggregateExprs = resolvedWindowAggregate.aggregateExpressions
+ aggregateExprs.foreach(validateAggregateExpression)
+ groupingExprs.foreach(validateGroupingExpression)
+
+ def validateAggregateExpression(expr: Expression): Unit = expr match {
+ // check no nested aggregation exists.
+ case aggExpr: Aggregation =>
+ aggExpr.children.foreach { child =>
+ child.preOrderVisit {
+ case agg: Aggregation =>
+ failValidation(
+ "It's not allowed to use an aggregate function as " +
+ "input of another aggregate function")
+ case _ => // ok
+ }
+ }
+ case a: Attribute if !groupingExprs.exists(_.checkEquals(a)) =>
+ failValidation(
+ s"Expression '$a' is invalid because it is neither" +
+ " present in group by nor an aggregate function")
+ case e if groupingExprs.exists(_.checkEquals(e)) => // ok
+ case e => e.children.foreach(validateAggregateExpression)
+ }
+
+ def validateGroupingExpression(expr: Expression): Unit = {
+ if (!expr.resultType.isKeyType) {
+ failValidation(
+ s"Expression $expr cannot be used as a grouping expression " +
+ "because it's not a valid key type which must be hashable and comparable")
+ }
+ }
+
+ // validate window
+ resolvedWindowAggregate.window.validate(tableEnv) match {
+ case ValidationFailure(msg) =>
+ failValidation(s"$window is invalid: $msg")
+ case ValidationSuccess => // ok
+ }
+
+ resolvedWindowAggregate
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/rel/LogicalWindowAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/rel/LogicalWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/rel/LogicalWindowAggregate.scala
new file mode 100644
index 0000000..9615168
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/logical/rel/LogicalWindowAggregate.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.logical.rel
+
+import java.util
+
+import org.apache.calcite.plan.{Convention, RelOptCluster, RelTraitSet}
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.{Aggregate, AggregateCall}
+import org.apache.calcite.rel.{RelNode, RelShuttle}
+import org.apache.calcite.util.ImmutableBitSet
+import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
+import org.apache.flink.api.table.FlinkTypeFactory
+import org.apache.flink.api.table.plan.logical.LogicalWindow
+
+class LogicalWindowAggregate(
+ window: LogicalWindow,
+ namedProperties: Seq[NamedWindowProperty],
+ cluster: RelOptCluster,
+ traitSet: RelTraitSet,
+ child: RelNode,
+ indicator: Boolean,
+ groupSet: ImmutableBitSet,
+ groupSets: util.List[ImmutableBitSet],
+ aggCalls: util.List[AggregateCall])
+ extends Aggregate(
+ cluster,
+ traitSet,
+ child,
+ indicator,
+ groupSet,
+ groupSets,
+ aggCalls) {
+
+ def getWindow = window
+
+ def getNamedProperties = namedProperties
+
+ override def copy(
+ traitSet: RelTraitSet,
+ input: RelNode,
+ indicator: Boolean,
+ groupSet: ImmutableBitSet,
+ groupSets: util.List[ImmutableBitSet],
+ aggCalls: util.List[AggregateCall])
+ : Aggregate = {
+
+ new LogicalWindowAggregate(
+ window,
+ namedProperties,
+ cluster,
+ traitSet,
+ input,
+ indicator,
+ groupSet,
+ groupSets,
+ aggCalls)
+ }
+
+ override def accept(shuttle: RelShuttle): RelNode = shuttle.visit(this)
+
+ override def deriveRowType(): RelDataType = {
+ val aggregateRowType = super.deriveRowType()
+ val typeFactory = getCluster.getTypeFactory.asInstanceOf[FlinkTypeFactory]
+ val builder = typeFactory.builder
+ builder.addAll(aggregateRowType.getFieldList)
+ namedProperties.foreach { namedProp =>
+ builder.add(
+ namedProp.name,
+ typeFactory.createTypeFromTypeInfo(namedProp.property.resultType)
+ )
+ }
+ builder.build()
+ }
+}
+
+object LogicalWindowAggregate {
+
+ def create(
+ window: LogicalWindow,
+ namedProperties: Seq[NamedWindowProperty],
+ aggregate: Aggregate)
+ : LogicalWindowAggregate = {
+
+ val cluster: RelOptCluster = aggregate.getCluster
+ val traitSet: RelTraitSet = cluster.traitSetOf(Convention.NONE)
+ new LogicalWindowAggregate(
+ window,
+ namedProperties,
+ cluster,
+ traitSet,
+ aggregate.getInput,
+ aggregate.indicator,
+ aggregate.getGroupSet,
+ aggregate.getGroupSets,
+ aggregate.getAggCallList)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkAggregate.scala
new file mode 100644
index 0000000..85129c4
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkAggregate.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.nodes
+
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.AggregateCall
+import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
+import org.apache.flink.api.table.runtime.aggregate.AggregateUtil._
+
+import scala.collection.JavaConverters._
+
+trait FlinkAggregate {
+
+ private[flink] def groupingToString(inputType: RelDataType, grouping: Array[Int]): String = {
+
+ val inFields = inputType.getFieldNames.asScala
+ grouping.map( inFields(_) ).mkString(", ")
+ }
+
+ private[flink] def aggregationToString(
+ inputType: RelDataType,
+ grouping: Array[Int],
+ rowType: RelDataType,
+ namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+ namedProperties: Seq[NamedWindowProperty])
+ : String = {
+
+ val inFields = inputType.getFieldNames.asScala
+ val outFields = rowType.getFieldNames.asScala
+
+ val groupStrings = grouping.map( inFields(_) )
+
+ val aggs = namedAggregates.map(_.getKey)
+ val aggStrings = aggs.map( a => s"${a.getAggregation}(${
+ if (a.getArgList.size() > 0) {
+ inFields(a.getArgList.get(0))
+ } else {
+ "*"
+ }
+ })")
+
+ val propStrings = namedProperties.map(_.property.toString)
+
+ (groupStrings ++ aggStrings ++ propStrings).zip(outFields).map {
+ case (f, o) => if (f == o) {
+ f
+ } else {
+ s"$f AS $o"
+ }
+ }.mkString(", ")
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkRel.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkRel.scala
index dad50a3..a4c7589 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/FlinkRel.scala
@@ -19,6 +19,12 @@
package org.apache.flink.api.table.plan.nodes
import org.apache.calcite.rex._
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.table.TableConfig
+import org.apache.flink.api.table.codegen.CodeGenerator
+import org.apache.flink.api.table.runtime.MapRunner
+
import scala.collection.JavaConversions._
trait FlinkRel {
@@ -44,4 +50,41 @@ trait FlinkRel {
case _ => throw new IllegalArgumentException("Unknown expression type: " + expr)
}
}
+
+ private[flink] def getConversionMapper(
+ config: TableConfig,
+ nullableInput: Boolean,
+ inputType: TypeInformation[Any],
+ expectedType: TypeInformation[Any],
+ conversionOperatorName: String,
+ fieldNames: Seq[String],
+ inputPojoFieldMapping: Option[Array[Int]] = None)
+ : MapFunction[Any, Any] = {
+
+ val generator = new CodeGenerator(
+ config,
+ nullableInput,
+ inputType,
+ None,
+ inputPojoFieldMapping)
+ val conversion = generator.generateConverterResultExpression(expectedType, fieldNames)
+
+ val body =
+ s"""
+ |${conversion.code}
+ |return ${conversion.resultTerm};
+ |""".stripMargin
+
+ val genFunction = generator.generateFunction(
+ conversionOperatorName,
+ classOf[MapFunction[Any, Any]],
+ body,
+ expectedType)
+
+ new MapRunner[Any, Any](
+ genFunction.name,
+ genFunction.code,
+ genFunction.returnType)
+
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetAggregate.scala
index c826d83..c73d781 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetAggregate.scala
@@ -25,10 +25,11 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.DataSet
+import org.apache.flink.api.table.plan.nodes.FlinkAggregate
import org.apache.flink.api.table.runtime.aggregate.AggregateUtil
import org.apache.flink.api.table.runtime.aggregate.AggregateUtil.CalcitePair
import org.apache.flink.api.table.typeutils.{RowTypeInfo, TypeConverter}
-import org.apache.flink.api.table.{FlinkTypeFactory, BatchTableEnvironment, Row}
+import org.apache.flink.api.table.{BatchTableEnvironment, FlinkTypeFactory, Row}
import scala.collection.JavaConverters._
@@ -38,12 +39,13 @@ import scala.collection.JavaConverters._
class DataSetAggregate(
cluster: RelOptCluster,
traitSet: RelTraitSet,
- input: RelNode,
+ inputNode: RelNode,
namedAggregates: Seq[CalcitePair[AggregateCall, String]],
rowRelDataType: RelDataType,
inputType: RelDataType,
grouping: Array[Int])
- extends SingleRel(cluster, traitSet, input)
+ extends SingleRel(cluster, traitSet, inputNode)
+ with FlinkAggregate
with DataSetRel {
override def deriveRowType() = rowRelDataType
@@ -61,16 +63,16 @@ class DataSetAggregate(
override def toString: String = {
s"Aggregate(${ if (!grouping.isEmpty) {
- s"groupBy: ($groupingToString), "
+ s"groupBy: (${groupingToString(inputType, grouping)}), "
} else {
""
- }}}select:($aggregationToString))"
+ }}select: (${aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil)}))"
}
override def explainTerms(pw: RelWriter): RelWriter = {
super.explainTerms(pw)
- .itemIf("groupBy",groupingToString, !grouping.isEmpty)
- .item("select", aggregationToString)
+ .itemIf("groupBy", groupingToString(inputType, grouping), !grouping.isEmpty)
+ .item("select", aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil))
}
override def computeSelfCost (planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = {
@@ -90,8 +92,11 @@ class DataSetAggregate(
val groupingKeys = grouping.indices.toArray
// add grouping fields, position keys in the input, and input type
- val aggregateResult = AggregateUtil.createOperatorFunctionsForAggregates(namedAggregates,
- inputType, getRowType, grouping, config)
+ val aggregateResult = AggregateUtil.createOperatorFunctionsForAggregates(
+ namedAggregates,
+ inputType,
+ getRowType,
+ grouping)
val inputDS = getInput.asInstanceOf[DataSetRel].translateToPlan(
tableEnv,
@@ -103,7 +108,7 @@ class DataSetAggregate(
.map(field => FlinkTypeFactory.toTypeInfo(field.getType))
.toArray
- val aggString = aggregationToString
+ val aggString = aggregationToString(inputType, grouping, getRowType, namedAggregates, Nil)
val prepareOpName = s"prepare select: ($aggString)"
val mappedInput = inputDS
.map(aggregateResult._1)
@@ -115,7 +120,8 @@ class DataSetAggregate(
val result = {
if (groupingKeys.length > 0) {
// grouped aggregation
- val aggOpName = s"groupBy: ($groupingToString), select:($aggString)"
+ val aggOpName = s"groupBy: (${groupingToString(inputType, grouping)}), " +
+ s"select: ($aggString)"
mappedInput.asInstanceOf[DataSet[Row]]
.groupBy(groupingKeys: _*)
@@ -151,36 +157,4 @@ class DataSetAggregate(
case _ => result
}
}
-
- private def groupingToString: String = {
-
- val inFields = inputType.getFieldNames.asScala
- grouping.map( inFields(_) ).mkString(", ")
- }
-
- private def aggregationToString: String = {
-
- val inFields = inputType.getFieldNames.asScala
- val outFields = getRowType.getFieldNames.asScala
-
- val groupStrings = grouping.map( inFields(_) )
-
- val aggs = namedAggregates.map(_.getKey)
- val aggStrings = aggs.map( a => s"${a.getAggregation}(${
- if (a.getArgList.size() > 0) {
- inFields(a.getArgList.get(0))
- } else {
- "*"
- }
- })")
-
- (groupStrings ++ aggStrings).zip(outFields).map {
- case (f, o) => if (f == o) {
- f
- } else {
- s"$f AS $o"
- }
- }.mkString(", ")
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetRel.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetRel.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetRel.scala
index 39532f0..82c75e1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetRel.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/dataset/DataSetRel.scala
@@ -71,40 +71,4 @@ trait DataSetRel extends RelNode with FlinkRel {
}
- private[dataset] def getConversionMapper(
- config: TableConfig,
- nullableInput: Boolean,
- inputType: TypeInformation[Any],
- expectedType: TypeInformation[Any],
- conversionOperatorName: String,
- fieldNames: Seq[String],
- inputPojoFieldMapping: Option[Array[Int]] = None): MapFunction[Any, Any] = {
-
- val generator = new CodeGenerator(
- config,
- nullableInput,
- inputType,
- None,
- inputPojoFieldMapping)
- val conversion = generator.generateConverterResultExpression(expectedType, fieldNames)
-
- val body =
- s"""
- |${conversion.code}
- |return ${conversion.resultTerm};
- |""".stripMargin
-
- val genFunction = generator.generateFunction(
- conversionOperatorName,
- classOf[MapFunction[Any, Any]],
- body,
- expectedType)
-
- new MapRunner[Any, Any](
- genFunction.name,
- genFunction.code,
- genFunction.returnType)
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
new file mode 100644
index 0000000..b9b4561
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/nodes/datastream/DataStreamAggregate.scala
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.nodes.datastream
+
+import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.rel.`type`.RelDataType
+import org.apache.calcite.rel.core.AggregateCall
+import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.tuple.Tuple
+import org.apache.flink.api.table.FlinkRelBuilder.NamedWindowProperty
+import org.apache.flink.api.table.expressions._
+import org.apache.flink.api.table.plan.logical._
+import org.apache.flink.api.table.plan.nodes.FlinkAggregate
+import org.apache.flink.api.table.plan.nodes.datastream.DataStreamAggregate.{createKeyedWindowedStream, createNonKeyedWindowedStream, transformToPropertyReads}
+import org.apache.flink.api.table.runtime.aggregate.AggregateUtil._
+import org.apache.flink.api.table.runtime.aggregate._
+import org.apache.flink.api.table.typeutils.TypeCheckUtils.isTimeInterval
+import org.apache.flink.api.table.typeutils.{RowIntervalTypeInfo, RowTypeInfo, TimeIntervalTypeInfo, TypeConverter}
+import org.apache.flink.api.table.{FlinkTypeFactory, Row, StreamTableEnvironment}
+import org.apache.flink.streaming.api.datastream.{AllWindowedStream, DataStream, KeyedStream, WindowedStream}
+import org.apache.flink.streaming.api.windowing.assigners._
+import org.apache.flink.streaming.api.windowing.time.Time
+import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
+
+import scala.collection.JavaConverters._
+
+class DataStreamAggregate(
+ window: LogicalWindow,
+ namedProperties: Seq[NamedWindowProperty],
+ cluster: RelOptCluster,
+ traitSet: RelTraitSet,
+ inputNode: RelNode,
+ namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+ rowRelDataType: RelDataType,
+ inputType: RelDataType,
+ grouping: Array[Int])
+ extends SingleRel(cluster, traitSet, inputNode)
+ with FlinkAggregate
+ with DataStreamRel {
+
+ override def deriveRowType() = rowRelDataType
+
+ override def copy(traitSet: RelTraitSet, inputs: java.util.List[RelNode]): RelNode = {
+ new DataStreamAggregate(
+ window,
+ namedProperties,
+ cluster,
+ traitSet,
+ inputs.get(0),
+ namedAggregates,
+ getRowType,
+ inputType,
+ grouping)
+ }
+
+ override def toString: String = {
+ s"Aggregate(${
+ if (!grouping.isEmpty) {
+ s"groupBy: (${groupingToString(inputType, grouping)}), "
+ } else {
+ ""
+ }
+ }window: ($window), " +
+ s"select: (${
+ aggregationToString(
+ inputType,
+ grouping,
+ getRowType,
+ namedAggregates,
+ namedProperties)
+ }))"
+ }
+
+ override def explainTerms(pw: RelWriter): RelWriter = {
+ super.explainTerms(pw)
+ .itemIf("groupBy", groupingToString(inputType, grouping), !grouping.isEmpty)
+ .item("window", window)
+ .item("select", aggregationToString(
+ inputType,
+ grouping,
+ getRowType,
+ namedAggregates,
+ namedProperties))
+ }
+
+ override def translateToPlan(
+ tableEnv: StreamTableEnvironment,
+ expectedType: Option[TypeInformation[Any]])
+ : DataStream[Any] = {
+
+ val config = tableEnv.getConfig
+
+ val groupingKeys = grouping.indices.toArray
+ // add grouping fields, position keys in the input, and input type
+ val aggregateResult = AggregateUtil.createOperatorFunctionsForAggregates(
+ namedAggregates,
+ inputType,
+ getRowType,
+ grouping)
+
+ val propertyReads = transformToPropertyReads(namedProperties.map(_.property))
+
+ val inputDS = input.asInstanceOf[DataStreamRel].translateToPlan(
+ tableEnv,
+ // tell the input operator that this operator currently only supports Rows as input
+ Some(TypeConverter.DEFAULT_ROW_TYPE))
+
+ // get the output types
+ val fieldTypes: Array[TypeInformation[_]] = getRowType.getFieldList.asScala
+ .map(field => FlinkTypeFactory.toTypeInfo(field.getType))
+ .toArray
+
+ val aggString = aggregationToString(
+ inputType,
+ grouping,
+ getRowType,
+ namedAggregates,
+ namedProperties)
+
+ val prepareOpName = s"prepare select: ($aggString)"
+ val mappedInput = inputDS
+ .map(aggregateResult._1)
+ .name(prepareOpName)
+
+ val groupReduceFunction = aggregateResult._2
+ val rowTypeInfo = new RowTypeInfo(fieldTypes)
+
+ val result = {
+ // grouped / keyed aggregation
+ if (groupingKeys.length > 0) {
+ val aggOpName = s"groupBy: (${groupingToString(inputType, grouping)}), " +
+ s"window: ($window), " +
+ s"select: ($aggString)"
+ val aggregateFunction = new AggregateWindowFunction(propertyReads, groupReduceFunction)
+
+ val keyedStream = mappedInput.keyBy(groupingKeys: _*)
+
+ val windowedStream = createKeyedWindowedStream(window, keyedStream)
+ .asInstanceOf[WindowedStream[Row, Tuple, DataStreamWindow]]
+
+ windowedStream
+ .apply(aggregateFunction)
+ .returns(rowTypeInfo)
+ .name(aggOpName)
+ .asInstanceOf[DataStream[Any]]
+ }
+ // global / non-keyed aggregation
+ else {
+ val aggOpName = s"window: ($window), select: ($aggString)"
+ val aggregateFunction = new AggregateAllWindowFunction(propertyReads, groupReduceFunction)
+
+ val windowedStream = createNonKeyedWindowedStream(window, mappedInput)
+ .asInstanceOf[AllWindowedStream[Row, DataStreamWindow]]
+
+ windowedStream
+ .apply(aggregateFunction)
+ .returns(rowTypeInfo)
+ .name(aggOpName)
+ .asInstanceOf[DataStream[Any]]
+ }
+ }
+
+ // if the expected type is not a Row, inject a mapper to convert to the expected type
+ expectedType match {
+ case Some(typeInfo) if typeInfo.getTypeClass != classOf[Row] =>
+ val mapName = s"convert: (${getRowType.getFieldNames.asScala.toList.mkString(", ")})"
+ result.map(getConversionMapper(
+ config = config,
+ nullableInput = false,
+ inputType = rowTypeInfo.asInstanceOf[TypeInformation[Any]],
+ expectedType = expectedType.get,
+ conversionOperatorName = "DataStreamAggregateConversion",
+ fieldNames = getRowType.getFieldNames.asScala
+ ))
+ .name(mapName)
+ case _ => result
+ }
+ }
+}
+
+object DataStreamAggregate {
+
+ private def transformToPropertyReads(namedProperties: Seq[WindowProperty])
+ : Array[WindowPropertyRead[_ <: Any]] = namedProperties.map {
+ case WindowStart(_) => new WindowStartRead()
+ case WindowEnd(_) => new WindowEndRead()
+ }.toArray
+
+ private def createKeyedWindowedStream(groupWindow: LogicalWindow, stream: KeyedStream[Row, Tuple])
+ : WindowedStream[Row, Tuple, _ <: DataStreamWindow] = groupWindow match {
+
+ case ProcessingTimeTumblingGroupWindow(_, size) if isTimeInterval(size.resultType) =>
+ stream.window(TumblingProcessingTimeWindows.of(asTime(size)))
+
+ case ProcessingTimeTumblingGroupWindow(_, size) =>
+ stream.countWindow(asCount(size))
+
+ case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
+ stream.window(TumblingEventTimeWindows.of(asTime(size)))
+
+ case EventTimeTumblingGroupWindow(_, _, size) =>
+ // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
+ // before applying the windowing logic. Otherwise, this would be the same as a
+ // ProcessingTimeTumblingGroupWindow
+ throw new UnsupportedOperationException("Event-time grouping windows on row intervals are " +
+ "currently not supported.")
+
+ case ProcessingTimeSlidingGroupWindow(_, size, slide) if isTimeInterval(size.resultType) =>
+ stream.window(SlidingProcessingTimeWindows.of(asTime(size), asTime(slide)))
+
+ case ProcessingTimeSlidingGroupWindow(_, size, slide) =>
+ stream.countWindow(asCount(size), asCount(slide))
+
+ case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
+ stream.window(SlidingEventTimeWindows.of(asTime(size), asTime(slide)))
+
+ case EventTimeSlidingGroupWindow(_, _, size, slide) =>
+ // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
+ // before applying the windowing logic. Otherwise, this would be the same as a
+ // ProcessingTimeTumblingGroupWindow
+ throw new UnsupportedOperationException("Event-time grouping windows on row intervals are " +
+ "currently not supported.")
+
+ case ProcessingTimeSessionGroupWindow(_, gap: Expression) =>
+ stream.window(ProcessingTimeSessionWindows.withGap(asTime(gap)))
+
+ case EventTimeSessionGroupWindow(_, _, gap) =>
+ stream.window(EventTimeSessionWindows.withGap(asTime(gap)))
+ }
+
+ private def createNonKeyedWindowedStream(groupWindow: LogicalWindow, stream: DataStream[Row])
+ : AllWindowedStream[Row, _ <: DataStreamWindow] = groupWindow match {
+
+ case ProcessingTimeTumblingGroupWindow(_, size) if isTimeInterval(size.resultType) =>
+ stream.windowAll(TumblingProcessingTimeWindows.of(asTime(size)))
+
+ case ProcessingTimeTumblingGroupWindow(_, size) =>
+ stream.countWindowAll(asCount(size))
+
+ case EventTimeTumblingGroupWindow(_, _, size) if isTimeInterval(size.resultType) =>
+ stream.windowAll(TumblingEventTimeWindows.of(asTime(size)))
+
+ case EventTimeTumblingGroupWindow(_, _, size) =>
+ // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
+ // before applying the windowing logic. Otherwise, this would be the same as a
+ // ProcessingTimeTumblingGroupWindow
+ throw new UnsupportedOperationException("Event-time grouping windows on row intervals are " +
+ "currently not supported.")
+
+ case ProcessingTimeSlidingGroupWindow(_, size, slide) if isTimeInterval(size.resultType) =>
+ stream.windowAll(SlidingProcessingTimeWindows.of(asTime(size), asTime(slide)))
+
+ case ProcessingTimeSlidingGroupWindow(_, size, slide) =>
+ stream.countWindowAll(asCount(size), asCount(slide))
+
+ case EventTimeSlidingGroupWindow(_, _, size, slide) if isTimeInterval(size.resultType) =>
+ stream.windowAll(SlidingEventTimeWindows.of(asTime(size), asTime(slide)))
+
+ case EventTimeSlidingGroupWindow(_, _, size, slide) =>
+ // TODO: EventTimeTumblingGroupWindow should sort the stream on event time
+ // before applying the windowing logic. Otherwise, this would be the same as a
+ // ProcessingTimeTumblingGroupWindow
+ throw new UnsupportedOperationException("Event-time grouping windows on row intervals are " +
+ "currently not supported.")
+
+ case ProcessingTimeSessionGroupWindow(_, gap) =>
+ stream.windowAll(ProcessingTimeSessionWindows.withGap(asTime(gap)))
+
+ case EventTimeSessionGroupWindow(_, _, gap) =>
+ stream.windowAll(EventTimeSessionWindows.withGap(asTime(gap)))
+ }
+
+ def asTime(expr: Expression): Time = expr match {
+ case Literal(value: Long, TimeIntervalTypeInfo.INTERVAL_MILLIS) => Time.milliseconds(value)
+ case _ => throw new IllegalArgumentException()
+ }
+
+ def asCount(expr: Expression): Long = expr match {
+ case Literal(value: Long, RowIntervalTypeInfo.INTERVAL_ROWS) => value
+ case _ => throw new IllegalArgumentException()
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/FlinkRuleSets.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/FlinkRuleSets.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/FlinkRuleSets.scala
index 3ed4385..638deac 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/FlinkRuleSets.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/FlinkRuleSets.scala
@@ -146,6 +146,7 @@ object FlinkRuleSets {
UnionEliminatorRule.INSTANCE,
// translate to DataStream nodes
+ DataStreamAggregateRule.INSTANCE,
DataStreamCalcRule.INSTANCE,
DataStreamScanRule.INSTANCE,
DataStreamUnionRule.INSTANCE,
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/dataSet/DataSetAggregateRule.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/dataSet/DataSetAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/dataSet/DataSetAggregateRule.scala
index 9f78adb..72ed27e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/dataSet/DataSetAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/dataSet/DataSetAggregateRule.scala
@@ -40,13 +40,13 @@ class DataSetAggregateRule
// check if we have distinct aggregates
val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
if (distinctAggs) {
- throw new TableException("DISTINCT aggregates are currently not supported.")
+ throw TableException("DISTINCT aggregates are currently not supported.")
}
// check if we have grouping sets
val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
if (groupSets || agg.indicator) {
- throw new TableException("GROUPING SETS are currently not supported.")
+ throw TableException("GROUPING SETS are currently not supported.")
}
!distinctAggs && !groupSets && !agg.indicator
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/datastream/DataStreamAggregateRule.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/datastream/DataStreamAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/datastream/DataStreamAggregateRule.scala
new file mode 100644
index 0000000..dff2adc
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/plan/rules/datastream/DataStreamAggregateRule.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.plan.rules.datastream
+
+import org.apache.calcite.plan.{Convention, RelOptRule, RelOptRuleCall, RelTraitSet}
+import org.apache.calcite.rel.RelNode
+import org.apache.calcite.rel.convert.ConverterRule
+import org.apache.flink.api.table.TableException
+import org.apache.flink.api.table.expressions.Alias
+import org.apache.flink.api.table.plan.logical.rel.LogicalWindowAggregate
+import org.apache.flink.api.table.plan.nodes.datastream.{DataStreamAggregate, DataStreamConvention}
+
+import scala.collection.JavaConversions._
+
+class DataStreamAggregateRule
+ extends ConverterRule(
+ classOf[LogicalWindowAggregate],
+ Convention.NONE,
+ DataStreamConvention.INSTANCE,
+ "DataStreamAggregateRule")
+ {
+
+ override def matches(call: RelOptRuleCall): Boolean = {
+ val agg: LogicalWindowAggregate = call.rel(0).asInstanceOf[LogicalWindowAggregate]
+
+ // check if we have distinct aggregates
+ val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
+ if (distinctAggs) {
+ throw TableException("DISTINCT aggregates are currently not supported.")
+ }
+
+ // check if we have grouping sets
+ val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
+ if (groupSets || agg.indicator) {
+ throw TableException("GROUPING SETS are currently not supported.")
+ }
+
+ !distinctAggs && !groupSets && !agg.indicator
+ }
+
+ override def convert(rel: RelNode): RelNode = {
+ val agg: LogicalWindowAggregate = rel.asInstanceOf[LogicalWindowAggregate]
+ val traitSet: RelTraitSet = rel.getTraitSet.replace(DataStreamConvention.INSTANCE)
+ val convInput: RelNode = RelOptRule.convert(agg.getInput, DataStreamConvention.INSTANCE)
+
+ new DataStreamAggregate(
+ agg.getWindow,
+ agg.getNamedProperties,
+ rel.getCluster,
+ traitSet,
+ convInput,
+ agg.getNamedAggCalls,
+ rel.getRowType,
+ agg.getInput.getRowType,
+ agg.getGroupSet.toArray)
+ }
+ }
+
+object DataStreamAggregateRule {
+ val INSTANCE: RelOptRule = new DataStreamAggregateRule
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
new file mode 100644
index 0000000..86f8a20
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateAllWindowFunction.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
+import org.apache.flink.api.table.Row
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.windowing.RichAllWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.Window
+import org.apache.flink.util.Collector
+
+class AggregateAllWindowFunction(
+ propertyReads: Array[WindowPropertyRead[_ <: Any]],
+ groupReduceFunction: RichGroupReduceFunction[Row, Row])
+ extends RichAllWindowFunction[Row, Row, Window] {
+
+ private var propertyCollector: PropertyCollector = _
+
+ override def open(parameters: Configuration): Unit = {
+ groupReduceFunction.open(parameters)
+ propertyCollector = new PropertyCollector(propertyReads)
+ }
+
+ override def apply(window: Window, input: Iterable[Row], out: Collector[Row]): Unit = {
+
+ // extract the properties from window
+ propertyReads.foreach(_.extract(window))
+
+ // set final collector
+ propertyCollector.finalCollector = out
+
+ // call wrapped reduce function with property collector
+ groupReduceFunction.reduce(input, propertyCollector)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceCombineFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceCombineFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceCombineFunction.scala
new file mode 100644
index 0000000..ca074cc
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceCombineFunction.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions.{CombineFunction, RichGroupReduceFunction}
+import org.apache.flink.api.table.Row
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.util.{Collector, Preconditions}
+
+import scala.collection.JavaConversions._
+
+
+/**
+ * It wraps the aggregate logic inside of
+ * [[org.apache.flink.api.java.operators.GroupReduceOperator]] and
+ * [[org.apache.flink.api.java.operators.GroupCombineOperator]]
+ *
+ * @param aggregates The aggregate functions.
+ * @param groupKeysMapping The index mapping of group keys between intermediate aggregate Row
+ * and output Row.
+ * @param aggregateMapping The index mapping between aggregate function list and aggregated value
+ * index in output Row.
+ */
+class AggregateReduceCombineFunction(
+ private val aggregates: Array[Aggregate[_ <: Any]],
+ private val groupKeysMapping: Array[(Int, Int)],
+ private val aggregateMapping: Array[(Int, Int)],
+ private val intermediateRowArity: Int,
+ private val finalRowArity: Int)
+ extends RichGroupReduceFunction[Row, Row] with CombineFunction[Row, Row] {
+
+ private var aggregateBuffer: Row = _
+ private var output: Row = _
+
+ override def open(config: Configuration): Unit = {
+ Preconditions.checkNotNull(aggregates)
+ Preconditions.checkNotNull(groupKeysMapping)
+ aggregateBuffer = new Row(intermediateRowArity)
+ output = new Row(finalRowArity)
+ }
+
+ /**
+ * For grouped intermediate aggregate Rows, merge all of them into aggregate buffer,
+ * calculate aggregated values output by aggregate buffer, and set them into output
+ * Row based on the mapping relation between intermediate aggregate Row and output Row.
+ *
+ * @param records Grouped intermediate aggregate Rows iterator.
+ * @param out The collector to hand results to.
+ *
+ */
+ override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
+
+ // Initiate intermediate aggregate value.
+ aggregates.foreach(_.initiate(aggregateBuffer))
+
+ // Merge intermediate aggregate value to buffer.
+ var last: Row = null
+ records.foreach((record) => {
+ aggregates.foreach(_.merge(record, aggregateBuffer))
+ last = record
+ })
+
+ // Set group keys value to final output.
+ groupKeysMapping.foreach {
+ case (after, previous) =>
+ output.setField(after, last.productElement(previous))
+ }
+
+ // Evaluate final aggregate value and set to output.
+ aggregateMapping.foreach {
+ case (after, previous) =>
+ output.setField(after, aggregates(previous).evaluate(aggregateBuffer))
+ }
+
+ out.collect(output)
+ }
+
+ /**
+ * For sub-grouped intermediate aggregate Rows, merge all of them into aggregate buffer,
+ *
+ * @param records Sub-grouped intermediate aggregate Rows iterator.
+ * @return Combined intermediate aggregate Row.
+ *
+ */
+ override def combine(records: Iterable[Row]): Row = {
+
+ // Initiate intermediate aggregate value.
+ aggregates.foreach(_.initiate(aggregateBuffer))
+
+ // Merge intermediate aggregate value to buffer.
+ var last: Row = null
+ records.foreach((record) => {
+ aggregates.foreach(_.merge(record, aggregateBuffer))
+ last = record
+ })
+
+ // Set group keys to aggregateBuffer.
+ for (i <- groupKeysMapping.indices) {
+ aggregateBuffer.setField(i, last.productElement(i))
+ }
+
+ aggregateBuffer
+ }
+}
[2/5] flink git commit: [FLINK-4691] [table] Add group-windows for
streaming tables to Table API.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceGroupFunction.scala
index 738953d..d81f3a1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateReduceGroupFunction.scala
@@ -19,11 +19,10 @@ package org.apache.flink.api.table.runtime.aggregate
import java.lang.Iterable
-import org.apache.flink.api.common.functions.{CombineFunction, RichGroupReduceFunction, RichMapPartitionFunction}
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.api.table.Row
import org.apache.flink.configuration.Configuration
-import org.apache.flink.util.Collector
-import org.apache.flink.util.Preconditions
+import org.apache.flink.util.{Collector, Preconditions}
import scala.collection.JavaConversions._
@@ -41,7 +40,8 @@ class AggregateReduceGroupFunction(
private val aggregates: Array[Aggregate[_ <: Any]],
private val groupKeysMapping: Array[(Int, Int)],
private val aggregateMapping: Array[(Int, Int)],
- private val intermediateRowArity: Int)
+ private val intermediateRowArity: Int,
+ private val finalRowArity: Int)
extends RichGroupReduceFunction[Row, Row] {
private var aggregateBuffer: Row = _
@@ -50,9 +50,8 @@ class AggregateReduceGroupFunction(
override def open(config: Configuration) {
Preconditions.checkNotNull(aggregates)
Preconditions.checkNotNull(groupKeysMapping)
- val finalRowLength: Int = groupKeysMapping.length + aggregateMapping.length
aggregateBuffer = new Row(intermediateRowArity)
- output = new Row(finalRowLength)
+ output = new Row(finalRowArity)
}
/**
@@ -77,13 +76,13 @@ class AggregateReduceGroupFunction(
})
// Set group keys value to final output.
- groupKeysMapping.map {
+ groupKeysMapping.foreach {
case (after, previous) =>
output.setField(after, last.productElement(previous))
}
// Evaluate final aggregate value and set to output.
- aggregateMapping.map {
+ aggregateMapping.foreach {
case (after, previous) =>
output.setField(after, aggregates(previous).evaluate(aggregateBuffer))
}
@@ -91,96 +90,3 @@ class AggregateReduceGroupFunction(
out.collect(output)
}
}
-
-/**
- * It wraps the aggregate logic inside of
- * [[org.apache.flink.api.java.operators.GroupReduceOperator]] and
- * [[org.apache.flink.api.java.operators.GroupCombineOperator]]
- *
- * @param aggregates The aggregate functions.
- * @param groupKeysMapping The index mapping of group keys between intermediate aggregate Row
- * and output Row.
- * @param aggregateMapping The index mapping between aggregate function list and aggregated value
- * index in output Row.
- */
-class AggregateReduceCombineFunction(
- private val aggregates: Array[Aggregate[_ <: Any]],
- private val groupKeysMapping: Array[(Int, Int)],
- private val aggregateMapping: Array[(Int, Int)],
- private val intermediateRowArity: Int)
- extends RichGroupReduceFunction[Row, Row] with CombineFunction[Row, Row] {
-
- private var aggregateBuffer: Row = _
- private var output: Row = _
-
- override def open(config: Configuration): Unit = {
- Preconditions.checkNotNull(aggregates)
- Preconditions.checkNotNull(groupKeysMapping)
- val finalRowLength: Int = groupKeysMapping.length + aggregateMapping.length
- aggregateBuffer = new Row(intermediateRowArity)
- output = new Row(finalRowLength)
- }
-
- /**
- * For grouped intermediate aggregate Rows, merge all of them into aggregate buffer,
- * calculate aggregated values output by aggregate buffer, and set them into output
- * Row based on the mapping relation between intermediate aggregate Row and output Row.
- *
- * @param records Grouped intermediate aggregate Rows iterator.
- * @param out The collector to hand results to.
- *
- */
- override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = {
-
- // Initiate intermediate aggregate value.
- aggregates.foreach(_.initiate(aggregateBuffer))
-
- // Merge intermediate aggregate value to buffer.
- var last: Row = null
- records.foreach((record) => {
- aggregates.foreach(_.merge(record, aggregateBuffer))
- last = record
- })
-
- // Set group keys value to final output.
- groupKeysMapping.map {
- case (after, previous) =>
- output.setField(after, last.productElement(previous))
- }
-
- // Evaluate final aggregate value and set to output.
- aggregateMapping.map {
- case (after, previous) =>
- output.setField(after, aggregates(previous).evaluate(aggregateBuffer))
- }
-
- out.collect(output)
- }
-
- /**
- * For sub-grouped intermediate aggregate Rows, merge all of them into aggregate buffer,
- *
- * @param records Sub-grouped intermediate aggregate Rows iterator.
- * @return Combined intermediate aggregate Row.
- *
- */
- override def combine(records: Iterable[Row]): Row = {
-
- // Initiate intermediate aggregate value.
- aggregates.foreach(_.initiate(aggregateBuffer))
-
- // Merge intermediate aggregate value to buffer.
- var last: Row = null
- records.foreach((record) => {
- aggregates.foreach(_.merge(record, aggregateBuffer))
- last = record
- })
-
- // Set group keys to aggregateBuffer.
- for (i <- 0 until groupKeysMapping.length) {
- aggregateBuffer.setField(i, last.productElement(i))
- }
-
- aggregateBuffer
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateUtil.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateUtil.scala
index 65d12c3..903cc07 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateUtil.scala
@@ -21,14 +21,14 @@ import java.util
import org.apache.calcite.rel.`type`._
import org.apache.calcite.rel.core.AggregateCall
-import org.apache.calcite.sql.SqlAggFunction
+import org.apache.calcite.sql.{SqlAggFunction, SqlKind}
import org.apache.calcite.sql.`type`.SqlTypeName._
import org.apache.calcite.sql.`type`.{SqlTypeFactoryImpl, SqlTypeName}
import org.apache.calcite.sql.fun._
-import org.apache.flink.api.common.functions.{GroupReduceFunction, MapFunction}
+import org.apache.flink.api.common.functions.{MapFunction, RichGroupReduceFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.table.typeutils.RowTypeInfo
-import org.apache.flink.api.table.{FlinkTypeFactory, Row, TableConfig, TableException}
+import org.apache.flink.api.table.{FlinkTypeFactory, Row, TableException}
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
@@ -61,10 +61,12 @@ object AggregateUtil {
* }}}
*
*/
- def createOperatorFunctionsForAggregates(namedAggregates: Seq[CalcitePair[AggregateCall, String]],
- inputType: RelDataType, outputType: RelDataType,
- groupings: Array[Int],
- config: TableConfig): (MapFunction[Any, Row], GroupReduceFunction[Row, Row] ) = {
+ def createOperatorFunctionsForAggregates(
+ namedAggregates: Seq[CalcitePair[AggregateCall, String]],
+ inputType: RelDataType,
+ outputType: RelDataType,
+ groupings: Array[Int])
+ : (MapFunction[Any, Row], RichGroupReduceFunction[Row, Row]) = {
val aggregateFunctionsAndFieldIndexes =
transformToAggregateFunctions(namedAggregates.map(_.getKey), inputType, groupings.length)
@@ -98,12 +100,20 @@ object AggregateUtil {
val reduceGroupFunction =
if (allPartialAggregate) {
- new AggregateReduceCombineFunction(aggregates, groupingOffsetMapping,
- aggOffsetMapping, intermediateRowArity)
+ new AggregateReduceCombineFunction(
+ aggregates,
+ groupingOffsetMapping,
+ aggOffsetMapping,
+ intermediateRowArity,
+ outputType.getFieldCount)
}
else {
- new AggregateReduceGroupFunction(aggregates, groupingOffsetMapping,
- aggOffsetMapping, intermediateRowArity)
+ new AggregateReduceGroupFunction(
+ aggregates,
+ groupingOffsetMapping,
+ aggOffsetMapping,
+ intermediateRowArity,
+ outputType.getFieldCount)
}
(mapFunction, reduceGroupFunction)
@@ -181,7 +191,7 @@ object AggregateUtil {
}
}
case sqlMinMaxFunction: SqlMinMaxAggFunction => {
- aggregates(index) = if (sqlMinMaxFunction.isMin) {
+ aggregates(index) = if (sqlMinMaxFunction.getKind == SqlKind.MIN) {
sqlTypeName match {
case TINYINT =>
new ByteMinAggregate
@@ -305,7 +315,7 @@ object AggregateUtil {
// input data, so if inputIndex is not -1, it must be a group key. Then we can
// find the field index in buffer data by the group keys index mapping between
// input data and buffer data.
- for (i <- 0 until groupKeys.length) {
+ for (i <- groupKeys.indices) {
if (inputIndex == groupKeys(i)) {
groupingOffsetMapping += ((outputIndex, i))
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
new file mode 100644
index 0000000..c65ac35
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/AggregateWindowFunction.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.lang.Iterable
+
+import org.apache.flink.api.common.functions.RichGroupReduceFunction
+import org.apache.flink.api.java.tuple.Tuple
+import org.apache.flink.api.table.Row
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.Window
+import org.apache.flink.util.Collector
+
+class AggregateWindowFunction(
+ propertyReads: Array[WindowPropertyRead[_ <: Any]],
+ groupReduceFunction: RichGroupReduceFunction[Row, Row])
+ extends RichWindowFunction[Row, Row, Tuple, Window] {
+
+ private var propertyCollector: PropertyCollector = _
+
+ override def open(parameters: Configuration): Unit = {
+ groupReduceFunction.open(parameters)
+ propertyCollector = new PropertyCollector(propertyReads)
+ }
+
+ override def apply(
+ key: Tuple,
+ window: Window,
+ input: Iterable[Row],
+ out: Collector[Row])
+ : Unit = {
+
+ // extract the properties from window
+ propertyReads.foreach(_.extract(window))
+
+ // set final collector
+ propertyCollector.finalCollector = out
+
+ // call wrapped reduce function with property collector
+ groupReduceFunction.reduce(input, propertyCollector)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
new file mode 100644
index 0000000..763ed0b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/PropertyCollector.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import org.apache.flink.api.table.Row
+import org.apache.flink.util.Collector
+
+/**
+ * Adds properties to the end of a row before it emits it to the final collector.
+ * The collector assumes that the row has placeholders at the end that can be filled.
+ */
+class PropertyCollector(properties: Array[WindowPropertyRead[_ <: Any]]) extends Collector[Row] {
+ var finalCollector: Collector[Row] = _
+
+ override def collect(record: Row): Unit = {
+ var i: Int = 0
+ while (i < properties.length) {
+ val idx = record.productArity - properties.length + i
+ record.setField(idx, properties(i).get())
+ i = i + 1
+ }
+ finalCollector.collect(record)
+ }
+
+ override def close(): Unit = finalCollector.close()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
new file mode 100644
index 0000000..dd12238
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowEndRead.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.sql.Timestamp
+
+import org.apache.calcite.runtime.SqlFunctions
+import org.apache.flink.streaming.api.windowing.windows.{TimeWindow, Window}
+
+class WindowEndRead extends WindowPropertyRead[Timestamp] {
+
+ private var ts: Timestamp = _
+
+ override def extract(window: Window): Unit = window match {
+ case timeWindow: TimeWindow =>
+ ts = SqlFunctions.internalToTimestamp(timeWindow.getEnd)
+ case _ =>
+ ts = null
+ }
+
+ override def get(): Timestamp = ts
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
new file mode 100644
index 0000000..88ea30d
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowPropertyRead.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import org.apache.flink.streaming.api.windowing.windows.Window
+
+/**
+ * Base class for reading a window property. The property will be extracted once and
+ * can be read multiple times.
+ */
+trait WindowPropertyRead[T] extends Serializable {
+
+ def extract(window: Window): Unit
+
+ def get(): T
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
new file mode 100644
index 0000000..d92cb18
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/runtime/aggregate/WindowStartRead.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.runtime.aggregate
+
+import java.sql.Timestamp
+
+import org.apache.calcite.runtime.SqlFunctions
+import org.apache.flink.streaming.api.windowing.windows.{TimeWindow, Window}
+
+class WindowStartRead extends WindowPropertyRead[Timestamp] {
+
+ private var ts: Timestamp = _
+
+ override def extract(window: Window): Unit = window match {
+ case timeWindow: TimeWindow =>
+ ts = SqlFunctions.internalToTimestamp(timeWindow.getStart)
+ case _ =>
+ ts = null
+ }
+
+ override def get(): Timestamp = ts
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
index 9d96780..ebb4dcb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/table.scala
@@ -20,8 +20,8 @@ package org.apache.flink.api.table
import org.apache.calcite.rel.RelNode
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.operators.join.JoinType
-import org.apache.flink.api.table.expressions.{Asc, ExpressionParser, UnresolvedAlias, Expression, Ordering}
-import org.apache.flink.api.table.plan.RexNodeTranslator._
+import org.apache.flink.api.table.expressions.{Asc, Expression, ExpressionParser, Ordering}
+import org.apache.flink.api.table.plan.ProjectionTranslator._
import org.apache.flink.api.table.plan.logical._
import org.apache.flink.api.table.sinks.TableSink
@@ -76,7 +76,7 @@ class Table(
* }}}
*/
def select(fields: Expression*): Table = {
- val projectionOnAggregates = fields.map(extractAggregations(_, tableEnv))
+ val projectionOnAggregates = fields.map(extractAggregationsAndProperties(_, tableEnv))
val aggregations = projectionOnAggregates.flatMap(_._2)
val projectList = expandProjectList(projectionOnAggregates.map(_._1), logicalPlan)
if (aggregations.nonEmpty) {
@@ -201,9 +201,6 @@ class Table(
* }}}
*/
def groupBy(fields: Expression*): GroupedTable = {
- if (tableEnv.isInstanceOf[StreamTableEnvironment]) {
- throw new ValidationException(s"Group by on stream tables is currently not supported.")
- }
new GroupedTable(this, fields)
}
@@ -632,6 +629,28 @@ class Table(
// emit the table to the configured table sink
tableEnv.writeToSink(this, configuredSink)
}
+
+ /**
+ * Groups the records of a table by assigning them to windows defined by a time or row interval.
+ *
+ * For streaming tables of infinite size, grouping into windows is required to define finite
+ * groups on which group-based aggregates can be computed.
+ *
+ * For batch tables of finite size, windowing essentially provides shortcuts for time-based
+ * groupBy.
+ *
+ * __Note__: window on non-grouped streaming table is a non-parallel operation, i.e., all data
+ * will be processed by a single operator.
+ *
+ * @param groupWindow group-window that specifies how elements are grouped.
+ * @return A windowed table.
+ */
+ def window(groupWindow: GroupWindow): GroupWindowedTable = {
+ if (tableEnv.isInstanceOf[BatchTableEnvironment]) {
+ throw new ValidationException(s"Windows on batch tables are currently not supported.")
+ }
+ new GroupWindowedTable(this, Seq(), groupWindow)
+ }
}
/**
@@ -653,19 +672,25 @@ class GroupedTable(
*/
def select(fields: Expression*): Table = {
- val projectionOnAggregates = fields.map(extractAggregations(_, table.tableEnv))
- val aggregations = projectionOnAggregates.flatMap(_._2)
+ val projectionOnAggsAndProps = fields.map(extractAggregationsAndProperties(_, table.tableEnv))
+ val aggregations = projectionOnAggsAndProps.flatMap(_._2)
+ val properties = projectionOnAggsAndProps.flatMap(_._3)
- val logical = if (aggregations.nonEmpty) {
- Project(projectionOnAggregates.map(e => UnresolvedAlias(e._1)),
- Aggregate(groupKey, aggregations, table.logicalPlan).validate(table.tableEnv)
- )
- } else {
- Project(projectionOnAggregates.map(e => UnresolvedAlias(e._1)),
- Aggregate(groupKey, Nil, table.logicalPlan).validate(table.tableEnv))
+ if (properties.nonEmpty) {
+ throw ValidationException("Window properties can only be used on windowed tables.")
}
- new Table(table.tableEnv, logical.validate(table.tableEnv))
+ val logical =
+ Project(
+ projectionOnAggsAndProps.map(e => UnresolvedAlias(e._1)),
+ Aggregate(
+ groupKey,
+ aggregations,
+ table.logicalPlan
+ ).validate(table.tableEnv)
+ ).validate(table.tableEnv)
+
+ new Table(table.tableEnv, logical)
}
/**
@@ -675,11 +700,84 @@ class GroupedTable(
* Example:
*
* {{{
- * tab.groupBy("key").select("key, value.avg + " The average" as average")
+ * tab.groupBy("key").select("key, value.avg + ' The average' as average")
* }}}
*/
def select(fields: String): Table = {
val fieldExprs = ExpressionParser.parseExpressionList(fields)
select(fieldExprs: _*)
}
+
+ /**
+ * Groups the records of a table by assigning them to windows defined by a time or row interval.
+ *
+ * For streaming tables of infinite size, grouping into windows is required to define finite
+ * groups on which group-based aggregates can be computed.
+ *
+ * For batch tables of finite size, windowing essentially provides shortcuts for time-based
+ * groupBy.
+ *
+ * @param groupWindow group-window that specifies how elements are grouped.
+ * @return A windowed table.
+ */
+ def window(groupWindow: GroupWindow): GroupWindowedTable = {
+ if (table.tableEnv.isInstanceOf[BatchTableEnvironment]) {
+ throw new ValidationException(s"Windows on batch tables are currently not supported.")
+ }
+ new GroupWindowedTable(table, groupKey, groupWindow)
+ }
+}
+
+class GroupWindowedTable(
+ private[flink] val table: Table,
+ private[flink] val groupKey: Seq[Expression],
+ private[flink] val window: GroupWindow) {
+
+ /**
+ * Performs a selection operation on a windowed table. Similar to an SQL SELECT statement.
+ * The field expressions can contain complex expressions and aggregations.
+ *
+ * Example:
+ *
+ * {{{
+ * groupWindowTable.select('key, 'window.start, 'value.avg + " The average" as 'average)
+ * }}}
+ */
+ def select(fields: Expression*): Table = {
+ val projectionOnAggsAndProps = fields.map(extractAggregationsAndProperties(_, table.tableEnv))
+ val aggregations = projectionOnAggsAndProps.flatMap(_._2)
+ val properties = projectionOnAggsAndProps.flatMap(_._3)
+
+ val groupWindow = window.toLogicalWindow
+
+ val logical =
+ Project(
+ projectionOnAggsAndProps.map(e => UnresolvedAlias(e._1)),
+ WindowAggregate(
+ groupKey,
+ groupWindow,
+ properties,
+ aggregations,
+ table.logicalPlan
+ ).validate(table.tableEnv)
+ ).validate(table.tableEnv)
+
+ new Table(table.tableEnv, logical)
+ }
+
+ /**
+ * Performs a selection operation on a group-windows table. Similar to an SQL SELECT statement.
+ * The field expressions can contain complex expressions and aggregations.
+ *
+ * Example:
+ *
+ * {{{
+ * groupWindowTable.select("key, window.start, value.avg + ' The average' as average")
+ * }}}
+ */
+ def select(fields: String): Table = {
+ val fieldExprs = ExpressionParser.parseExpressionList(fields)
+ select(fieldExprs: _*)
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/InternalTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/InternalTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/InternalTypeInfo.scala
new file mode 100644
index 0000000..2f896ec
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/InternalTypeInfo.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.typeutils
+
+import java.util.Objects
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.util.Preconditions._
+
+/**
+ * TypeInformation for internal types of the Table API that are for translation purposes only
+ * and should not be contained in final plan.
+ */
+@SerialVersionUID(-13064574364925255L)
+abstract class InternalTypeInfo[T](val clazz: Class[T])
+ extends TypeInformation[T]
+ with AtomicType[T] {
+
+ checkNotNull(clazz)
+
+ override def isBasicType: Boolean =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def isTupleType: Boolean =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def getArity: Int =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def getTotalFields: Int =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def getTypeClass: Class[T] = clazz
+
+ override def isKeyType: Boolean =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def createSerializer(config: ExecutionConfig): TypeSerializer[T] =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ override def createComparator(
+ sortOrderAscending: Boolean,
+ executionConfig: ExecutionConfig)
+ : TypeComparator[T] =
+ throw new UnsupportedOperationException("This type is for internal use only.")
+
+ // ----------------------------------------------------------------------------------------------
+
+ override def hashCode: Int = Objects.hash(clazz)
+
+ def canEqual(obj: Any): Boolean
+
+ override def equals(obj: Any): Boolean = {
+ obj match {
+ case other: InternalTypeInfo[_] =>
+ other.canEqual(this) && (this.clazz eq other.clazz)
+ case _ =>
+ false
+ }
+ }
+
+ override def toString: String = s"InternalTypeInfo"
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/IntervalTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/IntervalTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/IntervalTypeInfo.scala
deleted file mode 100644
index bf5cb58..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/IntervalTypeInfo.scala
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.table.typeutils
-
-import java.util.Objects
-
-import org.apache.flink.api.common.ExecutionConfig
-import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
-import org.apache.flink.api.common.typeutils.base.{IntComparator, IntSerializer, LongComparator, LongSerializer}
-import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
-import org.apache.flink.api.table.typeutils.IntervalTypeInfo.instantiateComparator
-import org.apache.flink.util.Preconditions._
-
-/**
- * TypeInformation for SQL INTERVAL types.
- */
-@SerialVersionUID(-1816179424364825258L)
-class IntervalTypeInfo[T](
- val clazz: Class[T],
- val serializer: TypeSerializer[T],
- val comparatorClass: Class[_ <: TypeComparator[T]])
- extends TypeInformation[T]
- with AtomicType[T] {
-
- checkNotNull(clazz)
- checkNotNull(serializer)
- checkNotNull(comparatorClass)
-
- override def isBasicType: Boolean = false
-
- override def isTupleType: Boolean = false
-
- override def getArity: Int = 1
-
- override def getTotalFields: Int = 1
-
- override def getTypeClass: Class[T] = clazz
-
- override def isKeyType: Boolean = true
-
- override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = serializer
-
- override def createComparator(
- sortOrderAscending: Boolean,
- executionConfig: ExecutionConfig)
- : TypeComparator[T] = instantiateComparator(comparatorClass, sortOrderAscending)
-
- // ----------------------------------------------------------------------------------------------
-
- override def hashCode: Int = Objects.hash(clazz, serializer, comparatorClass)
-
- def canEqual(obj: Any): Boolean = obj.isInstanceOf[IntervalTypeInfo[_]]
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case other: IntervalTypeInfo[_] =>
- other.canEqual(this) &&
- (this.clazz eq other.clazz) &&
- serializer == other.serializer &&
- (this.comparatorClass eq other.comparatorClass)
- case _ =>
- false
- }
- }
-
- override def toString: String = s"IntervalTypeInfo(${clazz.getSimpleName})"
-}
-
-object IntervalTypeInfo {
-
- val INTERVAL_MONTHS =
- new IntervalTypeInfo(classOf[java.lang.Integer], IntSerializer.INSTANCE, classOf[IntComparator])
-
- val INTERVAL_MILLIS =
- new IntervalTypeInfo(classOf[java.lang.Long], LongSerializer.INSTANCE, classOf[LongComparator])
-
- // ----------------------------------------------------------------------------------------------
-
- private def instantiateComparator[X](
- comparatorClass: Class[_ <: TypeComparator[X]],
- ascendingOrder: java.lang.Boolean)
- : TypeComparator[X] = {
- try {
- val constructor = comparatorClass.getConstructor(java.lang.Boolean.TYPE)
- constructor.newInstance(ascendingOrder)
- } catch {
- case e: Exception =>
- throw new RuntimeException(
- s"Could not initialize comparator ${comparatorClass.getName}", e)
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/RowIntervalTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/RowIntervalTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/RowIntervalTypeInfo.scala
new file mode 100644
index 0000000..4dc83d0
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/RowIntervalTypeInfo.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.typeutils
+
+/**
+ * TypeInformation for row intervals.
+ */
+@SerialVersionUID(-1306179424364925258L)
+class RowIntervalTypeInfo extends InternalTypeInfo[Long](classOf[Long]) {
+
+ def canEqual(obj: Any): Boolean = obj.isInstanceOf[RowIntervalTypeInfo]
+
+ override def toString: String = s"RowIntervalTypeInfo"
+}
+
+object RowIntervalTypeInfo {
+
+ val INTERVAL_ROWS = new RowIntervalTypeInfo()
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TimeIntervalTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TimeIntervalTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TimeIntervalTypeInfo.scala
new file mode 100644
index 0000000..b516745
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TimeIntervalTypeInfo.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table.typeutils
+
+import java.util.Objects
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.{AtomicType, TypeInformation}
+import org.apache.flink.api.common.typeutils.base.{IntComparator, IntSerializer, LongComparator, LongSerializer}
+import org.apache.flink.api.common.typeutils.{TypeComparator, TypeSerializer}
+import org.apache.flink.api.table.typeutils.TimeIntervalTypeInfo.instantiateComparator
+import org.apache.flink.util.Preconditions._
+
+/**
+ * TypeInformation for SQL INTERVAL types.
+ */
+@SerialVersionUID(-1816179424364825258L)
+class TimeIntervalTypeInfo[T](
+ val clazz: Class[T],
+ val serializer: TypeSerializer[T],
+ val comparatorClass: Class[_ <: TypeComparator[T]])
+ extends TypeInformation[T]
+ with AtomicType[T] {
+
+ checkNotNull(clazz)
+ checkNotNull(serializer)
+ checkNotNull(comparatorClass)
+
+ override def isBasicType: Boolean = false
+
+ override def isTupleType: Boolean = false
+
+ override def getArity: Int = 1
+
+ override def getTotalFields: Int = 1
+
+ override def getTypeClass: Class[T] = clazz
+
+ override def isKeyType: Boolean = true
+
+ override def createSerializer(config: ExecutionConfig): TypeSerializer[T] = serializer
+
+ override def createComparator(
+ sortOrderAscending: Boolean,
+ executionConfig: ExecutionConfig)
+ : TypeComparator[T] = instantiateComparator(comparatorClass, sortOrderAscending)
+
+ // ----------------------------------------------------------------------------------------------
+
+ override def hashCode: Int = Objects.hash(clazz, serializer, comparatorClass)
+
+ def canEqual(obj: Any): Boolean = obj.isInstanceOf[TimeIntervalTypeInfo[_]]
+
+ override def equals(obj: Any): Boolean = {
+ obj match {
+ case other: TimeIntervalTypeInfo[_] =>
+ other.canEqual(this) &&
+ (this.clazz eq other.clazz) &&
+ serializer == other.serializer &&
+ (this.comparatorClass eq other.comparatorClass)
+ case _ =>
+ false
+ }
+ }
+
+ override def toString: String = s"TimeIntervalTypeInfo(${clazz.getSimpleName})"
+}
+
+object TimeIntervalTypeInfo {
+
+ val INTERVAL_MONTHS = new TimeIntervalTypeInfo(
+ classOf[java.lang.Integer],
+ IntSerializer.INSTANCE,
+ classOf[IntComparator])
+
+ val INTERVAL_MILLIS = new TimeIntervalTypeInfo(
+ classOf[java.lang.Long],
+ LongSerializer.INSTANCE,
+ classOf[LongComparator])
+
+ // ----------------------------------------------------------------------------------------------
+
+ private def instantiateComparator[X](
+ comparatorClass: Class[_ <: TypeComparator[X]],
+ ascendingOrder: java.lang.Boolean)
+ : TypeComparator[X] = {
+ try {
+ val constructor = comparatorClass.getConstructor(java.lang.Boolean.TYPE)
+ constructor.newInstance(ascendingOrder)
+ } catch {
+ case e: Exception =>
+ throw new RuntimeException(
+ s"Could not initialize comparator ${comparatorClass.getName}", e)
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCheckUtils.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCheckUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCheckUtils.scala
index 4ff7888..aa8614b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCheckUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCheckUtils.scala
@@ -30,7 +30,7 @@ object TypeCheckUtils {
def isAdvanced(dataType: TypeInformation[_]): Boolean = dataType match {
case _: BasicTypeInfo[_] => false
case _: SqlTimeTypeInfo[_] => false
- case _: IntervalTypeInfo[_] => false
+ case _: TimeIntervalTypeInfo[_] => false
case _ => true
}
@@ -53,7 +53,7 @@ object TypeCheckUtils {
dataType.isInstanceOf[SqlTimeTypeInfo[_]]
def isTimeInterval(dataType: TypeInformation[_]): Boolean =
- dataType.isInstanceOf[IntervalTypeInfo[_]]
+ dataType.isInstanceOf[TimeIntervalTypeInfo[_]]
def isString(dataType: TypeInformation[_]): Boolean = dataType == STRING_TYPE_INFO
@@ -67,7 +67,7 @@ object TypeCheckUtils {
def assertNumericExpr(
dataType: TypeInformation[_],
caller: String)
- : ExprValidationResult = dataType match {
+ : ValidationResult = dataType match {
case _: NumericTypeInfo[_] =>
ValidationSuccess
case BIG_DEC_TYPE_INFO =>
@@ -76,7 +76,7 @@ object TypeCheckUtils {
ValidationFailure(s"$caller requires numeric types, get $dataType here")
}
- def assertOrderableExpr(dataType: TypeInformation[_], caller: String): ExprValidationResult = {
+ def assertOrderableExpr(dataType: TypeInformation[_], caller: String): ValidationResult = {
if (dataType.isSortKeyType) {
ValidationSuccess
} else {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCoercion.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCoercion.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCoercion.scala
index bb3d060..23154a5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCoercion.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/typeutils/TypeCoercion.scala
@@ -45,8 +45,8 @@ object TypeCoercion {
case (_, BIG_DEC_TYPE_INFO) => Some(BIG_DEC_TYPE_INFO)
case (BIG_DEC_TYPE_INFO, _) => Some(BIG_DEC_TYPE_INFO)
- case (stti: SqlTimeTypeInfo[_], _: IntervalTypeInfo[_]) => Some(stti)
- case (_: IntervalTypeInfo[_], stti: SqlTimeTypeInfo[_]) => Some(stti)
+ case (stti: SqlTimeTypeInfo[_], _: TimeIntervalTypeInfo[_]) => Some(stti)
+ case (_: TimeIntervalTypeInfo[_], stti: SqlTimeTypeInfo[_]) => Some(stti)
case tuple if tuple.productIterator.forall(numericWideningPrecedence.contains) =>
val higherIndex = numericWideningPrecedence.lastIndexWhere(t => t == tp1 || t == tp2)
@@ -103,8 +103,8 @@ object TypeCoercion {
case (INT_TYPE_INFO, SqlTimeTypeInfo.DATE) => true
case (INT_TYPE_INFO, SqlTimeTypeInfo.TIME) => true
case (LONG_TYPE_INFO, SqlTimeTypeInfo.TIMESTAMP) => true
- case (INT_TYPE_INFO, IntervalTypeInfo.INTERVAL_MONTHS) => true
- case (LONG_TYPE_INFO, IntervalTypeInfo.INTERVAL_MILLIS) => true
+ case (INT_TYPE_INFO, TimeIntervalTypeInfo.INTERVAL_MONTHS) => true
+ case (LONG_TYPE_INFO, TimeIntervalTypeInfo.INTERVAL_MILLIS) => true
case (SqlTimeTypeInfo.DATE, SqlTimeTypeInfo.TIME) => false
case (SqlTimeTypeInfo.TIME, SqlTimeTypeInfo.DATE) => false
@@ -113,8 +113,8 @@ object TypeCoercion {
case (SqlTimeTypeInfo.TIME, INT_TYPE_INFO) => true
case (SqlTimeTypeInfo.TIMESTAMP, LONG_TYPE_INFO) => true
- case (IntervalTypeInfo.INTERVAL_MONTHS, INT_TYPE_INFO) => true
- case (IntervalTypeInfo.INTERVAL_MILLIS, LONG_TYPE_INFO) => true
+ case (TimeIntervalTypeInfo.INTERVAL_MONTHS, INT_TYPE_INFO) => true
+ case (TimeIntervalTypeInfo.INTERVAL_MILLIS, LONG_TYPE_INFO) => true
case _ => false
}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ExprValidationResult.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ExprValidationResult.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ExprValidationResult.scala
deleted file mode 100644
index 8571051..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ExprValidationResult.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.table.validate
-
-/**
- * Represents the result of `Expression.validateInput`.
- */
-sealed trait ExprValidationResult {
- def isFailure: Boolean = !isSuccess
- def isSuccess: Boolean
-}
-
-/**
- * Represents the successful result of `Expression.checkInputDataTypes`.
- */
-object ValidationSuccess extends ExprValidationResult {
- val isSuccess: Boolean = true
-}
-
-/**
- * Represents the failing result of `Expression.checkInputDataTypes`,
- * with a error message to show the reason of failure.
- */
-case class ValidationFailure(message: String) extends ExprValidationResult {
- val isSuccess: Boolean = false
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ValidationResult.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ValidationResult.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ValidationResult.scala
new file mode 100644
index 0000000..5cc7d03
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/validate/ValidationResult.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.table.validate
+
+/**
+ * Represents the result of a validation.
+ */
+sealed trait ValidationResult {
+ def isFailure: Boolean = !isSuccess
+ def isSuccess: Boolean
+
+ /**
+ * Allows constructing a cascade of validation results.
+ * The first failure result will be returned.
+ */
+ def orElse(other: ValidationResult): ValidationResult = {
+ if (isSuccess) {
+ other
+ } else {
+ this
+ }
+ }
+}
+
+/**
+ * Represents the successful result of a validation.
+ */
+object ValidationSuccess extends ValidationResult {
+ val isSuccess: Boolean = true
+}
+
+/**
+ * Represents the failing result of a validation,
+ * with a error message to show the reason of failure.
+ */
+case class ValidationFailure(message: String) extends ValidationResult {
+ val isSuccess: Boolean = false
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/windows.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/windows.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/windows.scala
new file mode 100644
index 0000000..32d67d7
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/api/table/windows.scala
@@ -0,0 +1,390 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.table
+
+import org.apache.flink.api.table.expressions.{Expression, ExpressionParser}
+import org.apache.flink.api.table.plan.logical._
+
+/**
+ * A group-window specification.
+ *
+ * Group-windows group rows based on time or row-count intervals and is therefore essentially a
+ * special type of groupBy. Just like groupBy, group-windows allow to compute aggregates
+ * on groups of elements.
+ *
+ * Infinite streaming tables can only be grouped into time or row intervals. Hence window grouping
+ * is required to apply aggregations on streaming tables.
+ *
+ * For finite batch tables, group-windows provide shortcuts for time-based groupBy.
+ *
+ */
+trait GroupWindow {
+
+ /**
+ * Converts an API class to a logical window for planning.
+ */
+ private[flink] def toLogicalWindow: LogicalWindow
+}
+
+/**
+ * A group-window operating on event-time.
+ *
+ * @param timeField defines the time mode for streaming tables. For batch table it defines the
+ * time attribute on which is grouped.
+ */
+abstract class EventTimeWindow(timeField: Expression) extends GroupWindow {
+
+ protected var name: Option[Expression] = None
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: Expression): EventTimeWindow = {
+ this.name = Some(alias)
+ this
+ }
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: String): EventTimeWindow = as(ExpressionParser.parseExpression(alias))
+}
+
+// ------------------------------------------------------------------------------------------------
+// Tumbling group-windows
+// ------------------------------------------------------------------------------------------------
+
+/**
+ * Tumbling group-window.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
+ * grouped by processing-time.
+ *
+ * @param size the size of the window either as time or row-count interval.
+ */
+class TumblingWindow(size: Expression) extends GroupWindow {
+
+ /**
+ * Tumbling group-window.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * @param size the size of the window either as time or row-count interval.
+ */
+ def this(size: String) = this(ExpressionParser.parseExpression(size))
+
+ private var alias: Option[Expression] = None
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a tumbling group-window on event-time
+ */
+ def on(timeField: Expression): TumblingEventTimeWindow =
+ new TumblingEventTimeWindow(alias, timeField, size)
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a tumbling group-window on event-time
+ */
+ def on(timeField: String): TumblingEventTimeWindow =
+ on(ExpressionParser.parseExpression(timeField))
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: Expression): TumblingWindow = {
+ this.alias = Some(alias)
+ this
+ }
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: String): TumblingWindow = as(ExpressionParser.parseExpression(alias))
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ ProcessingTimeTumblingGroupWindow(alias, size)
+}
+
+/**
+ * Tumbling group-window on event-time.
+ */
+class TumblingEventTimeWindow(
+ alias: Option[Expression],
+ time: Expression,
+ size: Expression)
+ extends EventTimeWindow(time) {
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ EventTimeTumblingGroupWindow(name.orElse(alias), time, size)
+}
+
+// ------------------------------------------------------------------------------------------------
+// Sliding group windows
+// ------------------------------------------------------------------------------------------------
+
+/**
+ * Partially specified sliding window.
+ *
+ * @param size the size of the window either as time or row-count interval.
+ */
+class SlideWithSize(size: Expression) {
+
+ /**
+ * Partially specified sliding window.
+ *
+ * @param size the size of the window either as time or row-count interval.
+ */
+ def this(size: String) = this(ExpressionParser.parseExpression(size))
+
+ /**
+ * Specifies the window's slide as time or row-count interval.
+ *
+ * The slide determines the interval in which windows are started. Hence, sliding windows can
+ * overlap if the slide is smaller than the size of the window.
+ *
+ * For example, you could have windows of size 15 minutes that slide by 3 minutes. With this
+ * 15 minutes worth of elements are grouped every 3 minutes and each row contributes to 5
+ * windows.
+ *
+ * @param slide the slide of the window either as time or row-count interval.
+ * @return a sliding group-window
+ */
+ def every(slide: Expression): SlidingWindow = new SlidingWindow(size, slide)
+
+ /**
+ * Specifies the window's slide as time or row-count interval.
+ *
+ * The slide determines the interval in which windows are started. Hence, sliding windows can
+ * overlap if the slide is smaller than the size of the window.
+ *
+ * For example, you could have windows of size 15 minutes that slide by 3 minutes. With this
+ * 15 minutes worth of elements are grouped every 3 minutes and each row contributes to 5
+ * windows.
+ *
+ * @param slide the slide of the window either as time or row-count interval.
+ * @return a sliding group-window
+ */
+ def every(slide: String): SlidingWindow = every(ExpressionParser.parseExpression(slide))
+}
+
+/**
+ * Sliding group-window.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
+ * grouped by processing-time.
+ *
+ * @param size the size of the window either as time or row-count interval.
+ */
+class SlidingWindow(
+ size: Expression,
+ slide: Expression)
+ extends GroupWindow {
+
+ private var alias: Option[Expression] = None
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a sliding group-window on event-time
+ */
+ def on(timeField: Expression): SlidingEventTimeWindow =
+ new SlidingEventTimeWindow(alias, timeField, size, slide)
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a sliding group-window on event-time
+ */
+ def on(timeField: String): SlidingEventTimeWindow =
+ on(ExpressionParser.parseExpression(timeField))
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: Expression): SlidingWindow = {
+ this.alias = Some(alias)
+ this
+ }
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: String): SlidingWindow = as(ExpressionParser.parseExpression(alias))
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ ProcessingTimeSlidingGroupWindow(alias, size, slide)
+}
+
+/**
+ * Sliding group-window on event-time.
+ */
+class SlidingEventTimeWindow(
+ alias: Option[Expression],
+ timeField: Expression,
+ size: Expression,
+ slide: Expression)
+ extends EventTimeWindow(timeField) {
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ EventTimeSlidingGroupWindow(name.orElse(alias), timeField, size, slide)
+}
+
+// ------------------------------------------------------------------------------------------------
+// Session group windows
+// ------------------------------------------------------------------------------------------------
+
+/**
+ * Session group-window.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows are
+ * grouped by processing-time.
+ *
+ * @param gap the time interval of inactivity before a window is closed.
+ */
+class SessionWindow(gap: Expression) extends GroupWindow {
+
+ /**
+ * Session group-window.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * @param gap the time interval of inactivity before a window is closed.
+ */
+ def this(gap: String) = this(ExpressionParser.parseExpression(gap))
+
+ private var alias: Option[Expression] = None
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a session group-window on event-time
+ */
+ def on(timeField: Expression): SessionEventTimeWindow =
+ new SessionEventTimeWindow(alias, timeField, gap)
+
+ /**
+ * Specifies the time attribute on which rows are grouped.
+ *
+ * For streaming tables call [[on('rowtime)]] to specify grouping by event-time. Otherwise rows
+ * are grouped by processing-time.
+ *
+ * For batch tables, refer to a timestamp or long attribute.
+ *
+ * @param timeField time mode for streaming tables and time attribute for batch tables
+ * @return a session group-window on event-time
+ */
+ def on(timeField: String): SessionEventTimeWindow =
+ on(ExpressionParser.parseExpression(timeField))
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: Expression): SessionWindow = {
+ this.alias = Some(alias)
+ this
+ }
+
+ /**
+ * Assigns an alias for this window that the following `select()` clause can refer to in order
+ * to access window properties such as window start or end time.
+ *
+ * @param alias alias for this window
+ * @return this window
+ */
+ def as(alias: String): SessionWindow = as(ExpressionParser.parseExpression(alias))
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ ProcessingTimeSessionGroupWindow(alias, gap)
+}
+
+/**
+ * Session group-window on event-time.
+ */
+class SessionEventTimeWindow(
+ alias: Option[Expression],
+ timeField: Expression,
+ gap: Expression)
+ extends EventTimeWindow(timeField) {
+
+ override private[flink] def toLogicalWindow: LogicalWindow =
+ EventTimeSessionGroupWindow(name.orElse(alias), timeField, gap)
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/TableEnvironmentITCase.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/TableEnvironmentITCase.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/TableEnvironmentITCase.java
index 2d662d6..294cba2 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/TableEnvironmentITCase.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/api/java/batch/TableEnvironmentITCase.java
@@ -118,7 +118,7 @@ public class TableEnvironmentITCase extends TableProgramsTestBase {
tableEnv.registerDataSet("MyTable", ds2);
}
- @Test(expected = ValidationException.class)
+ @Test(expected = TableException.class)
public void testScanUnregisteredTable() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
@@ -146,7 +146,7 @@ public class TableEnvironmentITCase extends TableProgramsTestBase {
compareResultAsText(results, expected);
}
- @Test(expected = ValidationException.class)
+ @Test(expected = TableException.class)
public void testIllegalName() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
BatchTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(env, config());
@@ -157,7 +157,7 @@ public class TableEnvironmentITCase extends TableProgramsTestBase {
tableEnv.registerTable("_DataSetTable_42", t);
}
- @Test(expected = ValidationException.class)
+ @Test(expected = TableException.class)
public void testRegisterTableFromOtherEnv() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
BatchTableEnvironment tableEnv1 = TableEnvironment.getTableEnvironment(env, config());
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/TableEnvironmentITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/TableEnvironmentITCase.scala
index 2d82dbc..4c07615 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/TableEnvironmentITCase.scala
@@ -91,7 +91,7 @@ class TableEnvironmentITCase(
tEnv.registerDataSet("MyTable", ds2)
}
- @Test(expected = classOf[ValidationException])
+ @Test(expected = classOf[TableException])
def testScanUnregisteredTable(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tEnv = TableEnvironment.getTableEnvironment(env, config)
@@ -132,7 +132,7 @@ class TableEnvironmentITCase(
tEnv.registerDataSet("MyTable", t2)
}
- @Test(expected = classOf[ValidationException])
+ @Test(expected = classOf[TableException])
def testRegisterTableFromOtherEnv(): Unit = {
val env = ExecutionEnvironment.getExecutionEnvironment
val tEnv1 = TableEnvironment.getTableEnvironment(env, config)
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/table/CalcITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/table/CalcITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/table/CalcITCase.scala
index 8a2612e..c3758a4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/table/CalcITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/batch/table/CalcITCase.scala
@@ -27,7 +27,7 @@ import org.apache.flink.api.scala.batch.utils.TableProgramsTestBase.TableConfigM
import org.apache.flink.api.scala.table._
import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.api.table.expressions.Literal
-import org.apache.flink.api.table.{Row, TableEnvironment, ValidationException}
+import org.apache.flink.api.table.{Row, TableEnvironment, TableException, ValidationException}
import org.apache.flink.test.util.MultipleProgramsTestBase.TestExecutionMode
import org.apache.flink.test.util.TestBaseUtils
import org.junit.Assert._
@@ -163,9 +163,9 @@ class CalcITCase(
try {
CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, '*, 'b, 'c)
- fail("ValidationException expected")
+ fail("TableException expected")
} catch {
- case _: ValidationException => //ignore
+ case _: TableException => //ignore
}
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/44f3977e/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
new file mode 100644
index 0000000..6bb513e
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/api/scala/stream/table/AggregationsITCase.scala
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.stream.table
+
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.stream.table.GroupWindowITCase.TimestampWithEqualWatermark
+import org.apache.flink.api.scala.stream.utils.StreamITCase
+import org.apache.flink.api.scala.table._
+import org.apache.flink.api.table.{Row, _}
+import org.apache.flink.streaming.api.TimeCharacteristic
+import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
+import org.junit.Assert._
+import org.junit.Test
+
+import scala.collection.mutable
+
+/**
+ * We only test some aggregations until better testing of constructed DataStream
+ * programs is possible.
+ */
+class AggregationsITCase extends StreamingMultipleProgramsTestBase {
+
+ val data = List(
+ (1L, 1, "Hi"),
+ (2L, 2, "Hello"),
+ (4L, 2, "Hello"),
+ (8L, 3, "Hello world"),
+ (16L, 3, "Hello world"))
+
+ @Test
+ def testProcessingTimeSlidingGroupWindowOverCount(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 2.rows every 1.rows)
+ .select('string, 'int.count)
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq("Hello world,1", "Hello world,2", "Hello,1", "Hello,2", "Hi,1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSessionGroupWindowOverTime(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Session withGap 7.milli on 'rowtime)
+ .select('string, 'int.count)
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq("Hello world,1", "Hello world,1", "Hello,2", "Hi,1")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testAllProcessingTimeTumblingGroupWindowOverCount(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env.fromCollection(data)
+ val table = stream.toTable(tEnv, 'long, 'int, 'string)
+
+ val windowedTable = table
+ .window(Tumble over 2.rows)
+ .select('int.count)
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq("2", "2")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeTumblingWindow(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Tumble over 5.milli on 'rowtime as 'w)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq(
+ "Hello world,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01",
+ "Hello world,1,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02",
+ "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+
+ @Test
+ def testEventTimeSlidingWindow(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.testResults = mutable.MutableList()
+
+ val stream = env
+ .fromCollection(data)
+ .assignTimestampsAndWatermarks(new TimestampWithEqualWatermark())
+ val table = stream.toTable(tEnv, 'long, 'int, 'string)
+
+ val windowedTable = table
+ .groupBy('string)
+ .window(Slide over 10.milli every 5.milli on 'rowtime as 'w)
+ .select('string, 'int.count, 'w.start, 'w.end)
+
+ val results = windowedTable.toDataStream[Row]
+ results.addSink(new StreamITCase.StringSink)
+ env.execute()
+
+ val expected = Seq(
+ "Hello world,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
+ "Hello world,1,1970-01-01 00:00:00.005,1970-01-01 00:00:00.015",
+ "Hello world,1,1970-01-01 00:00:00.01,1970-01-01 00:00:00.02",
+ "Hello world,1,1970-01-01 00:00:00.015,1970-01-01 00:00:00.025",
+ "Hello,2,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
+ "Hello,2,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01",
+ "Hi,1,1969-12-31 23:59:59.995,1970-01-01 00:00:00.005",
+ "Hi,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.01")
+ assertEquals(expected.sorted, StreamITCase.testResults.sorted)
+ }
+}
+
+object GroupWindowITCase {
+ class TimestampWithEqualWatermark extends AssignerWithPunctuatedWatermarks[(Long, Int, String)] {
+
+ override def checkAndGetNextWatermark(
+ lastElement: (Long, Int, String),
+ extractedTimestamp: Long)
+ : Watermark = {
+ new Watermark(extractedTimestamp)
+ }
+
+ override def extractTimestamp(
+ element: (Long, Int, String),
+ previousElementTimestamp: Long): Long = {
+ element._1
+ }
+ }
+}