You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2017/08/29 20:11:08 UTC
[1/5] flink git commit: [FLINK-7309] [table] Fix NullPointerException
when selecting null fields.
Repository: flink
Updated Branches:
refs/heads/master 88848e74c -> df7452d98
[FLINK-7309] [table] Fix NullPointerException when selecting null fields.
This closes #4479.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/29e849b1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/29e849b1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/29e849b1
Branch: refs/heads/master
Commit: 29e849b1bf9180a5aa5f2d500efb283a39839caa
Parents: 68fdaa5
Author: Yestin <87...@qq.com>
Authored: Fri Aug 4 17:21:03 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Aug 29 22:10:17 2017 +0200
----------------------------------------------------------------------
.../flink/table/codegen/CodeGenerator.scala | 6 ++--
.../table/expressions/TemporalTypesTest.scala | 32 ++++++++++++++++++--
2 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/29e849b1/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 154e8ad..bf6ee21 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -1211,15 +1211,15 @@ abstract class CodeGenerator(
}
val wrappedCode = if (nullCheck && !isReference(fieldType)) {
+ // assumes that fieldType is a boxed primitive.
s"""
- |$tmpTypeTerm $tmpTerm = $unboxedFieldCode;
- |boolean $nullTerm = $tmpTerm == null;
+ |boolean $nullTerm = $fieldTerm == null;
|$resultTypeTerm $resultTerm;
|if ($nullTerm) {
| $resultTerm = $defaultValue;
|}
|else {
- | $resultTerm = $tmpTerm;
+ | $resultTerm = $fieldTerm;
|}
|""".stripMargin
} else if (nullCheck) {
http://git-wip-us.apache.org/repos/asf/flink/blob/29e849b1/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
index 1d761c3..8fae11a 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/TemporalTypesTest.scala
@@ -538,10 +538,31 @@ class TemporalTypesTest extends ExpressionTestBase {
"1990-09-12 10:20:45.123")
}
+ @Test
+ def testSelectNullValues(): Unit ={
+ testAllApis(
+ 'f11,
+ "f11",
+ "f11",
+ "null"
+ )
+ testAllApis(
+ 'f12,
+ "f12",
+ "f12",
+ "null"
+ )
+ testAllApis(
+ 'f13,
+ "f13",
+ "f13",
+ "null"
+ )
+ }
// ----------------------------------------------------------------------------------------------
def testData: Row = {
- val testData = new Row(11)
+ val testData = new Row(14)
testData.setField(0, Date.valueOf("1990-10-14"))
testData.setField(1, Time.valueOf("10:20:45"))
testData.setField(2, Timestamp.valueOf("1990-10-14 10:20:45.123"))
@@ -553,6 +574,10 @@ class TemporalTypesTest extends ExpressionTestBase {
testData.setField(8, 1467012213000L)
testData.setField(9, 24)
testData.setField(10, 12000L)
+ // null selection test.
+ testData.setField(11, null)
+ testData.setField(12, null)
+ testData.setField(13, null)
testData
}
@@ -568,6 +593,9 @@ class TemporalTypesTest extends ExpressionTestBase {
Types.INT,
Types.LONG,
Types.INTERVAL_MONTHS,
- Types.INTERVAL_MILLIS).asInstanceOf[TypeInformation[Any]]
+ Types.INTERVAL_MILLIS,
+ Types.SQL_DATE,
+ Types.SQL_TIME,
+ Types.SQL_TIMESTAMP).asInstanceOf[TypeInformation[Any]]
}
}
[2/5] flink git commit: [FLINK-7245] [table] Add operators to hold
back watermarks with static delays.
Posted by fh...@apache.org.
[FLINK-7245] [table] Add operators to hold back watermarks with static delays.
This closes #4530.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/68fdaa57
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/68fdaa57
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/68fdaa57
Branch: refs/heads/master
Commit: 68fdaa57e35b8ee30a262aad4d26926b18054c57
Parents: 1fc0b64
Author: Xingcan Cui <xi...@gmail.com>
Authored: Wed Aug 9 20:54:16 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Aug 29 22:10:17 2017 +0200
----------------------------------------------------------------------
...yedCoProcessOperatorWithWatermarkDelay.scala | 58 ++++++++++++
...KeyedProcessOperatorWithWatermarkDelay.scala | 59 +++++++++++++
...oProcessOperatorWithWatermarkDelayTest.scala | 93 ++++++++++++++++++++
...dProcessOperatorWithWatermarkDelayTest.scala | 78 ++++++++++++++++
.../api/operators/AbstractStreamOperator.java | 2 +-
.../operators/InternalTimeServiceManager.java | 2 +-
6 files changed, 290 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelay.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelay.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelay.scala
new file mode 100644
index 0000000..f25de25
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelay.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.operators
+
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
+import org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator
+import org.apache.flink.streaming.api.watermark.Watermark
+
+/**
+ * A [[KeyedCoProcessOperator]] that supports holding back watermarks with a static delay.
+ */
+class KeyedCoProcessOperatorWithWatermarkDelay[KEY, IN1, IN2, OUT](
+ private val flatMapper: CoProcessFunction[IN1, IN2, OUT],
+ private val watermarkDelay: Long = 0L)
+ extends KeyedCoProcessOperator[KEY, IN1, IN2, OUT](flatMapper) {
+
+ /** emits watermark without delay */
+ def emitWithoutDelay(mark: Watermark): Unit = output.emitWatermark(mark)
+
+ /** emits watermark with delay */
+ def emitWithDelay(mark: Watermark): Unit = {
+ output.emitWatermark(new Watermark(mark.getTimestamp - watermarkDelay))
+ }
+
+ if (watermarkDelay < 0) {
+ throw new IllegalArgumentException("The watermark delay should be non-negative.")
+ }
+
+ // choose watermark emitter
+ val emitter: Watermark => Unit = if (watermarkDelay == 0) {
+ emitWithoutDelay
+ } else {
+ emitWithDelay
+ }
+
+ @throws[Exception]
+ override def processWatermark(mark: Watermark) {
+ if (timeServiceManager != null) timeServiceManager.advanceWatermark(mark)
+
+ emitter(mark)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala
new file mode 100644
index 0000000..74b4773
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelay.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators
+
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator
+import org.apache.flink.streaming.api.watermark.Watermark
+
+/**
+ * A [[KeyedProcessOperator]] that supports holding back watermarks with a static delay.
+ */
+class KeyedProcessOperatorWithWatermarkDelay[KEY, IN, OUT](
+ private val function: ProcessFunction[IN, OUT],
+ private var watermarkDelay: Long = 0L)
+ extends KeyedProcessOperator[KEY, IN, OUT](function) {
+
+ /** emits watermark without delay */
+ def emitWithoutDelay(mark: Watermark): Unit = output.emitWatermark(mark)
+
+ /** emits watermark with delay */
+ def emitWithDelay(mark: Watermark): Unit = {
+ output.emitWatermark(new Watermark(mark.getTimestamp - watermarkDelay))
+ }
+
+ if (watermarkDelay < 0) {
+ throw new IllegalArgumentException("The watermark delay should be non-negative.")
+ }
+
+ // choose watermark emitter
+ val emitter: Watermark => Unit = if (watermarkDelay == 0) {
+ emitWithoutDelay
+ } else {
+ emitWithDelay
+ }
+
+ @throws[Exception]
+ override def processWatermark(mark: Watermark) {
+ if (timeServiceManager != null) timeServiceManager.advanceWatermark(mark)
+
+ emitter(mark)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelayTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelayTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelayTest.scala
new file mode 100644
index 0000000..243a034
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedCoProcessOperatorWithWatermarkDelayTest.scala
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.runtime.operators
+
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.{KeyedTwoInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.util.{Collector, TestLogger}
+import org.junit.Test
+
+/**
+ * Tests [[KeyedCoProcessOperatorWithWatermarkDelay]].
+ */
+class KeyedCoProcessOperatorWithWatermarkDelayTest extends TestLogger {
+
+ @Test
+ def testHoldingBackWatermarks(): Unit = {
+ val operator = new KeyedCoProcessOperatorWithWatermarkDelay[String, Integer, String, String](
+ new EmptyCoProcessFunction, 100)
+ val testHarness = new KeyedTwoInputStreamOperatorTestHarness[String, Integer, String, String](
+ operator,
+ new IntToStringKeySelector, new CoIdentityKeySelector[String],
+ BasicTypeInfo.STRING_TYPE_INFO)
+
+ testHarness.setup()
+ testHarness.open()
+ testHarness.processWatermark1(new Watermark(101))
+ testHarness.processWatermark2(new Watermark(202))
+ testHarness.processWatermark1(new Watermark(103))
+ testHarness.processWatermark2(new Watermark(204))
+
+ val expectedOutput = new ConcurrentLinkedQueue[AnyRef]
+ expectedOutput.add(new Watermark(1))
+ expectedOutput.add(new Watermark(3))
+
+ TestHarnessUtil.assertOutputEquals(
+ "Output was not correct.",
+ expectedOutput,
+ testHarness.getOutput)
+
+ testHarness.close()
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testDelayParameter(): Unit = {
+ new KeyedCoProcessOperatorWithWatermarkDelay[AnyRef, Integer, String, String](
+ new EmptyCoProcessFunction, -1)
+ }
+}
+
+private class EmptyCoProcessFunction extends CoProcessFunction[Integer, String, String] {
+ override def processElement1(
+ value: Integer,
+ ctx: CoProcessFunction[Integer, String, String]#Context,
+ out: Collector[String]): Unit = {
+ // do nothing
+ }
+
+ override def processElement2(
+ value: String,
+ ctx: CoProcessFunction[Integer, String, String]#Context,
+ out: Collector[String]): Unit = {
+ //do nothing
+ }
+}
+
+
+private class IntToStringKeySelector extends KeySelector[Integer, String] {
+ override def getKey(value: Integer): String = String.valueOf(value)
+}
+
+private class CoIdentityKeySelector[T] extends KeySelector[T, T] {
+ override def getKey(value: T): T = value
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelayTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelayTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelayTest.scala
new file mode 100644
index 0000000..d419453
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessOperatorWithWatermarkDelayTest.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators
+
+import java.util.concurrent.ConcurrentLinkedQueue
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.streaming.api.watermark.Watermark
+import org.apache.flink.streaming.util.{KeyedOneInputStreamOperatorTestHarness, TestHarnessUtil}
+import org.apache.flink.util.{Collector, TestLogger}
+import org.junit.Test
+
+/**
+ * Tests [[KeyedProcessOperatorWithWatermarkDelay]].
+ */
+class KeyedProcessOperatorWithWatermarkDelayTest extends TestLogger {
+
+ @Test
+ def testHoldingBackWatermarks(): Unit = {
+ val operator = new KeyedProcessOperatorWithWatermarkDelay[Integer, Integer, String](
+ new EmptyProcessFunction, 100)
+ val testHarness = new KeyedOneInputStreamOperatorTestHarness[Integer, Integer, String](
+ operator, new IdentityKeySelector, BasicTypeInfo.INT_TYPE_INFO)
+
+ testHarness.setup()
+ testHarness.open()
+ testHarness.processWatermark(new Watermark(101))
+ testHarness.processWatermark(new Watermark(103))
+
+ val expectedOutput = new ConcurrentLinkedQueue[AnyRef]
+ expectedOutput.add(new Watermark(1))
+ expectedOutput.add(new Watermark(3))
+
+ TestHarnessUtil.assertOutputEquals(
+ "Output was not correct.",
+ expectedOutput,
+ testHarness.getOutput)
+
+ testHarness.close()
+ }
+
+ @Test(expected = classOf[IllegalArgumentException])
+ def testDelayParameter(): Unit = {
+ new KeyedProcessOperatorWithWatermarkDelay[Integer, Integer, String](
+ new EmptyProcessFunction, -1)
+ }
+}
+
+private class EmptyProcessFunction extends ProcessFunction[Integer, String] {
+ override def processElement(
+ value: Integer,
+ ctx: ProcessFunction[Integer, String]#Context,
+ out: Collector[String]): Unit = {
+ // do nothing
+ }
+}
+
+private class IdentityKeySelector[T] extends KeySelector[T, T] {
+ override def getKey(value: T): T = value
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index a28fc30..fc043a8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -160,7 +160,7 @@ public abstract class AbstractStreamOperator<OUT>
// ---------------- time handler ------------------
- private transient InternalTimeServiceManager<?, ?> timeServiceManager;
+ protected transient InternalTimeServiceManager<?, ?> timeServiceManager;
// ---------------- two-input operator watermarks ------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/68fdaa57/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
index 17af3aa..7d5cb91 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/InternalTimeServiceManager.java
@@ -44,7 +44,7 @@ import java.util.Map;
* @param <N> The type of namespace used for the timers.
*/
@Internal
-class InternalTimeServiceManager<K, N> {
+public class InternalTimeServiceManager<K, N> {
private final int totalKeyGroups;
private final KeyGroupsList localKeyGroupRange;
[5/5] flink git commit: [FLINK-7398] [table] Add Logging trait to
prevent serialization of Logger.
Posted by fh...@apache.org.
[FLINK-7398] [table] Add Logging trait to prevent serialization of Logger.
This closes #4576.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/df7452d9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/df7452d9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/df7452d9
Branch: refs/heads/master
Commit: df7452d9811b0aa88919d7e3c1f6c34b36ac9b38
Parents: 29e849b
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Aug 23 14:13:07 2017 -0700
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Aug 29 22:10:18 2017 +0200
----------------------------------------------------------------------
.../table/catalog/ExternalCatalogSchema.scala | 8 +++---
.../table/catalog/ExternalTableSourceUtil.scala | 6 ++---
.../datastream/DataStreamGroupAggregate.scala | 11 ++++----
.../DataStreamGroupWindowAggregate.scala | 10 +++----
.../datastream/DataStreamOverAggregate.scala | 16 +++++------
.../runtime/CRowCorrelateProcessRunner.scala | 7 +++--
.../flink/table/runtime/CRowMapRunner.scala | 7 +++--
.../table/runtime/CRowOutputProcessRunner.scala | 7 +++--
.../flink/table/runtime/CRowProcessRunner.scala | 7 +++--
.../table/runtime/CorrelateFlatMapRunner.scala | 7 +++--
.../flink/table/runtime/FlatJoinRunner.scala | 9 +++----
.../flink/table/runtime/FlatMapRunner.scala | 9 +++----
.../apache/flink/table/runtime/MapRunner.scala | 9 +++----
.../flink/table/runtime/MapSideJoinRunner.scala | 9 +++----
.../aggregate/AggregateAggFunction.scala | 5 ++--
.../runtime/aggregate/DataSetAggFunction.scala | 5 ++--
.../aggregate/DataSetFinalAggFunction.scala | 5 ++--
.../aggregate/DataSetPreAggFunction.scala | 6 ++---
...SetSessionWindowAggReduceGroupFunction.scala | 8 +++---
...aSetSessionWindowAggregatePreProcessor.scala | 8 +++---
...tSlideTimeWindowAggReduceGroupFunction.scala | 6 ++---
...taSetSlideWindowAggReduceGroupFunction.scala | 6 ++---
...umbleCountWindowAggReduceGroupFunction.scala | 6 ++---
...TumbleTimeWindowAggReduceGroupFunction.scala | 6 ++---
.../aggregate/DataSetWindowAggMapFunction.scala | 6 ++---
.../aggregate/GroupAggProcessFunction.scala | 14 +++++-----
.../aggregate/ProcTimeBoundedRangeOver.scala | 6 ++---
.../aggregate/ProcTimeBoundedRowsOver.scala | 21 +++++++--------
.../aggregate/ProcTimeUnboundedOver.scala | 14 +++++-----
.../aggregate/RowTimeBoundedRangeOver.scala | 6 ++---
.../aggregate/RowTimeBoundedRowsOver.scala | 7 +++--
.../aggregate/RowTimeUnboundedOver.scala | 7 +++--
.../conversion/CRowToJavaTupleMapRunner.scala | 7 +++--
.../conversion/CRowToScalaTupleMapRunner.scala | 7 +++--
.../runtime/io/CRowValuesInputFormat.scala | 7 +++--
.../table/runtime/io/ValuesInputFormat.scala | 9 +++----
.../runtime/join/ProcTimeWindowInnerJoin.scala | 7 +++--
.../org/apache/flink/table/util/Logging.scala | 28 ++++++++++++++++++++
38 files changed, 165 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogSchema.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogSchema.scala
index 197449c..c74066f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogSchema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogSchema.scala
@@ -18,12 +18,12 @@
package org.apache.flink.table.catalog
-import java.util.{Collections => JCollections, Collection => JCollection, LinkedHashSet => JLinkedHashSet, Set => JSet}
+import java.util.{Collection => JCollection, Collections => JCollections, LinkedHashSet => JLinkedHashSet, Set => JSet}
import org.apache.calcite.linq4j.tree.Expression
import org.apache.calcite.schema._
import org.apache.flink.table.api.{CatalogNotExistException, TableNotExistException}
-import org.slf4j.{Logger, LoggerFactory}
+import org.apache.flink.table.util.Logging
import scala.collection.JavaConverters._
@@ -38,9 +38,7 @@ import scala.collection.JavaConverters._
*/
class ExternalCatalogSchema(
catalogIdentifier: String,
- catalog: ExternalCatalog) extends Schema {
-
- private val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ catalog: ExternalCatalog) extends Schema with Logging {
/**
* Looks up a sub-schema by the given sub-schema name in the external catalog.
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
index ccc2e9e..6bacac1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
@@ -27,9 +27,9 @@ import org.apache.flink.table.api.{AmbiguousTableSourceConverterException, NoMat
import org.apache.flink.table.plan.schema.{StreamTableSourceTable, TableSourceTable}
import org.apache.flink.table.plan.stats.FlinkStatistic
import org.apache.flink.table.sources.{StreamTableSource, TableSource}
+import org.apache.flink.table.util.Logging
import org.apache.flink.util.InstantiationUtil
import org.reflections.Reflections
-import org.slf4j.{Logger, LoggerFactory}
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -37,13 +37,11 @@ import scala.collection.mutable
/**
* The utility class is used to convert ExternalCatalogTable to TableSourceTable.
*/
-object ExternalTableSourceUtil {
+object ExternalTableSourceUtil extends Logging {
// config file to specify scan package to search TableSourceConverter
private val tableSourceConverterConfigFileName = "tableSourceConverter.properties"
- private val LOG: Logger = LoggerFactory.getLogger(this.getClass)
-
// registered table type with the TableSourceConverter.
// Key is table type name, Value is set of converter class.
private val tableTypeToTableSourceConvertersClazz = {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
index 590d9be..58c9d82 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupAggregate.scala
@@ -24,13 +24,13 @@ import org.apache.flink.api.java.functions.NullByteKeySelector
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment}
import org.apache.flink.table.codegen.AggregationCodeGenerator
-import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.plan.nodes.CommonAggregate
-import org.apache.flink.table.plan.schema.RowSchema
import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
+import org.apache.flink.table.plan.schema.RowSchema
import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
+import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
/**
*
@@ -55,9 +55,8 @@ class DataStreamGroupAggregate(
groupings: Array[Int])
extends SingleRel(cluster, traitSet, inputNode)
with CommonAggregate
- with DataStreamRel {
-
- private val LOG = LoggerFactory.getLogger(this.getClass)
+ with DataStreamRel
+ with Logging {
override def deriveRowType() = schema.relDataType
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 0cf86f7..b15350f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -29,7 +29,6 @@ import org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger
import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableException}
import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
-import org.apache.flink.table.calcite.FlinkTypeFactory
import org.apache.flink.table.codegen.AggregationCodeGenerator
import org.apache.flink.table.expressions.ExpressionUtils._
import org.apache.flink.table.expressions.ResolvedFieldReference
@@ -44,7 +43,7 @@ import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
import org.apache.flink.table.typeutils.TypeCheckUtils.isTimeInterval
import org.apache.flink.table.runtime.triggers.StateCleaningCountTrigger
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
class DataStreamGroupWindowAggregate(
window: LogicalWindow,
@@ -56,9 +55,10 @@ class DataStreamGroupWindowAggregate(
schema: RowSchema,
inputSchema: RowSchema,
grouping: Array[Int])
- extends SingleRel(cluster, traitSet, inputNode) with CommonAggregate with DataStreamRel {
-
- private val LOG = LoggerFactory.getLogger(this.getClass)
+ extends SingleRel(cluster, traitSet, inputNode)
+ with CommonAggregate
+ with DataStreamRel
+ with Logging {
override def deriveRowType(): RelDataType = schema.relDataType
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
index 7bf342a..6234525 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamOverAggregate.scala
@@ -20,23 +20,23 @@ package org.apache.flink.table.plan.nodes.datastream
import java.util.{List => JList}
import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rel.core.Window.Group
import org.apache.calcite.rel.core.{AggregateCall, Window}
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
-import org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING
+import org.apache.flink.api.java.functions.NullByteKeySelector
import org.apache.flink.streaming.api.datastream.DataStream
import org.apache.flink.table.api.{StreamQueryConfig, StreamTableEnvironment, TableException}
import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.table.plan.nodes.OverAggregate
-import org.apache.flink.table.plan.schema.RowSchema
-import org.apache.flink.table.runtime.aggregate._
-import org.apache.flink.api.java.functions.NullByteKeySelector
import org.apache.flink.table.codegen.AggregationCodeGenerator
+import org.apache.flink.table.plan.nodes.OverAggregate
import org.apache.flink.table.plan.rules.datastream.DataStreamRetractionRules
+import org.apache.flink.table.plan.schema.RowSchema
import org.apache.flink.table.runtime.aggregate.AggregateUtil.CalcitePair
+import org.apache.flink.table.runtime.aggregate._
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
class DataStreamOverAggregate(
logicWindow: Window,
@@ -47,8 +47,8 @@ class DataStreamOverAggregate(
inputSchema: RowSchema)
extends SingleRel(cluster, traitSet, inputNode)
with OverAggregate
- with DataStreamRel {
- private val LOG = LoggerFactory.getLogger(this.getClass)
+ with DataStreamRel
+ with Logging {
override def deriveRowType(): RelDataType = schema.relDataType
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
index 4f0a785..2553d9c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowCorrelateProcessRunner.scala
@@ -25,9 +25,9 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.{Logger, LoggerFactory}
/**
* A CorrelateProcessRunner with [[CRow]] input and [[CRow]] output.
@@ -40,9 +40,8 @@ class CRowCorrelateProcessRunner(
@transient var returnType: TypeInformation[CRow])
extends ProcessFunction[CRow, CRow]
with ResultTypeQueryable[CRow]
- with Compiler[Any] {
-
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ with Compiler[Any]
+ with Logging {
private var function: ProcessFunction[Row, Row] = _
private var collector: TableFunctionCollector[_] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala
index 9ed9188..54bac60 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowMapRunner.scala
@@ -24,8 +24,8 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.LoggerFactory
/**
* MapRunner with [[CRow]] input.
@@ -36,9 +36,8 @@ class CRowMapRunner[OUT](
@transient var returnType: TypeInformation[OUT])
extends RichMapFunction[CRow, OUT]
with ResultTypeQueryable[OUT]
- with Compiler[MapFunction[Row, OUT]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[MapFunction[Row, OUT]]
+ with Logging {
private var function: MapFunction[Row, OUT] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala
index c80f291..600b898 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowOutputProcessRunner.scala
@@ -25,9 +25,9 @@ import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.operators.TimestampedCollector
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* ProcessRunner with [[CRow]] output.
@@ -38,9 +38,8 @@ class CRowOutputProcessRunner(
@transient var returnType: TypeInformation[CRow])
extends ProcessFunction[Any, CRow]
with ResultTypeQueryable[CRow]
- with Compiler[ProcessFunction[Any, Row]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[ProcessFunction[Any, Row]]
+ with Logging {
private var function: ProcessFunction[Any, Row] = _
private var cRowWrapper: CRowWrappingCollector = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
index cef62a5..a7f3d72 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CRowProcessRunner.scala
@@ -25,9 +25,9 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* ProcessRunner with [[CRow]] input and [[CRow]] output.
@@ -38,9 +38,8 @@ class CRowProcessRunner(
@transient var returnType: TypeInformation[CRow])
extends ProcessFunction[CRow, CRow]
with ResultTypeQueryable[CRow]
- with Compiler[ProcessFunction[Row, Row]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[ProcessFunction[Row, Row]]
+ with Logging {
private var function: ProcessFunction[Row, Row] = _
private var cRowWrapper: CRowWrappingCollector = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
index 478b6b6..e2f5e61 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/CorrelateFlatMapRunner.scala
@@ -24,8 +24,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
import org.apache.flink.util.Collector
-import org.slf4j.{Logger, LoggerFactory}
class CorrelateFlatMapRunner[IN, OUT](
flatMapName: String,
@@ -35,9 +35,8 @@ class CorrelateFlatMapRunner[IN, OUT](
@transient var returnType: TypeInformation[OUT])
extends RichFlatMapFunction[IN, OUT]
with ResultTypeQueryable[OUT]
- with Compiler[Any] {
-
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ with Compiler[Any]
+ with Logging {
private var function: FlatMapFunction[IN, OUT] = _
private var collector: TableFunctionCollector[_] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
index 67acc0b..0bf6569 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatJoinRunner.scala
@@ -21,10 +21,10 @@ package org.apache.flink.table.runtime
import org.apache.flink.api.common.functions.{FlatJoinFunction, RichFlatJoinFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.table.codegen.Compiler
import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
class FlatJoinRunner[IN1, IN2, OUT](
name: String,
@@ -32,9 +32,8 @@ class FlatJoinRunner[IN1, IN2, OUT](
@transient var returnType: TypeInformation[OUT])
extends RichFlatJoinFunction[IN1, IN2, OUT]
with ResultTypeQueryable[OUT]
- with Compiler[FlatJoinFunction[IN1, IN2, OUT]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[FlatJoinFunction[IN1, IN2, OUT]]
+ with Logging {
private var function: FlatJoinFunction[IN1, IN2, OUT] = null
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
index 938da59..6c1f804 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/FlatMapRunner.scala
@@ -22,11 +22,11 @@ import org.apache.flink.api.common.functions.util.FunctionUtils
import org.apache.flink.api.common.functions.{FlatMapFunction, RichFlatMapFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.table.codegen.Compiler
import org.apache.flink.configuration.Configuration
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
class FlatMapRunner(
name: String,
@@ -34,9 +34,8 @@ class FlatMapRunner(
@transient var returnType: TypeInformation[Row])
extends RichFlatMapFunction[Row, Row]
with ResultTypeQueryable[Row]
- with Compiler[FlatMapFunction[Row, Row]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[FlatMapFunction[Row, Row]]
+ with Logging {
private var function: FlatMapFunction[Row, Row] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
index 14eeecf..00d18ec 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapRunner.scala
@@ -21,9 +21,9 @@ package org.apache.flink.table.runtime
import org.apache.flink.api.common.functions.{MapFunction, RichMapFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.table.codegen.Compiler
import org.apache.flink.configuration.Configuration
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
class MapRunner[IN, OUT](
name: String,
@@ -31,9 +31,8 @@ class MapRunner[IN, OUT](
@transient var returnType: TypeInformation[OUT])
extends RichMapFunction[IN, OUT]
with ResultTypeQueryable[OUT]
- with Compiler[MapFunction[IN, OUT]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[MapFunction[IN, OUT]]
+ with Logging {
private var function: MapFunction[IN, OUT] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
index 00b7b8e..5f5a2cc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/MapSideJoinRunner.scala
@@ -21,9 +21,9 @@ package org.apache.flink.table.runtime
import org.apache.flink.api.common.functions.{FlatJoinFunction, RichFlatMapFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.table.codegen.Compiler
import org.apache.flink.configuration.Configuration
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
abstract class MapSideJoinRunner[IN1, IN2, SINGLE_IN, MULTI_IN, OUT](
name: String,
@@ -32,9 +32,8 @@ abstract class MapSideJoinRunner[IN1, IN2, SINGLE_IN, MULTI_IN, OUT](
broadcastSetName: String)
extends RichFlatMapFunction[MULTI_IN, OUT]
with ResultTypeQueryable[OUT]
- with Compiler[FlatJoinFunction[IN1, IN2, OUT]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[FlatJoinFunction[IN1, IN2, OUT]]
+ with Logging {
protected var function: FlatJoinFunction[IN1, IN2, OUT] = _
protected var broadcastSet: Option[SINGLE_IN] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
index dd9c015..d3bffda 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateAggFunction.scala
@@ -21,8 +21,8 @@ package org.apache.flink.table.runtime.aggregate
import org.apache.flink.api.common.functions.AggregateFunction
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.LoggerFactory
/**
* Aggregate Function used for the aggregate operator in
@@ -31,9 +31,8 @@ import org.slf4j.LoggerFactory
* @param genAggregations Generated aggregate helper function
*/
class AggregateAggFunction(genAggregations: GeneratedAggregationsFunction)
- extends AggregateFunction[CRow, Row, Row] with Compiler[GeneratedAggregations] {
+ extends AggregateFunction[CRow, Row, Row] with Compiler[GeneratedAggregations] with Logging {
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def createAccumulator(): Row = {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
index 5f459f9..83e1b13 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetAggFunction.scala
@@ -22,9 +22,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* [[RichGroupReduceFunction]] to compute aggregates that do not support pre-aggregation for batch
@@ -35,12 +35,11 @@ import org.slf4j.LoggerFactory
class DataSetAggFunction(
private val genAggregations: GeneratedAggregationsFunction)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations] with Logging {
private var output: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
index 9b81992..5276271 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetFinalAggFunction.scala
@@ -23,9 +23,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* [[RichGroupReduceFunction]] to compute the final result of a pre-aggregated aggregation
@@ -36,12 +36,11 @@ import org.slf4j.LoggerFactory
class DataSetFinalAggFunction(
private val genAggregations: GeneratedAggregationsFunction)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations] with Logging {
private var output: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
index 8febe3e..fc3366b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetPreAggFunction.scala
@@ -22,9 +22,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions._
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* [[GroupCombineFunction]] and [[MapPartitionFunction]] to compute pre-aggregates for batch
@@ -36,12 +36,12 @@ class DataSetPreAggFunction(genAggregations: GeneratedAggregationsFunction)
extends AbstractRichFunction
with GroupCombineFunction[Row, Row]
with MapPartitionFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var output: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
index 2160ef5..d99ca31 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggReduceGroupFunction.scala
@@ -20,11 +20,11 @@ package org.apache.flink.table.runtime.aggregate
import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
-import org.apache.flink.types.Row
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
+import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* It wraps the aggregate logic inside of
@@ -54,7 +54,8 @@ class DataSetSessionWindowAggReduceGroupFunction(
gap: Long,
isInputCombined: Boolean)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var collector: RowTimeWindowPropertyCollector = _
private val intermediateRowWindowStartPos = keysAndAggregatesArity
@@ -63,7 +64,6 @@ class DataSetSessionWindowAggReduceGroupFunction(
private var output: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
index 9bcac30..666bfee 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSessionWindowAggregatePreProcessor.scala
@@ -22,11 +22,11 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.{AbstractRichFunction, GroupCombineFunction, MapPartitionFunction}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.types.Row
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
+import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* This wraps the aggregate logic inside of
@@ -46,13 +46,13 @@ class DataSetSessionWindowAggregatePreProcessor(
with MapPartitionFunction[Row,Row]
with GroupCombineFunction[Row,Row]
with ResultTypeQueryable[Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var output: Row = _
private val rowTimeFieldPos = keysAndAggregatesArity
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
index b3a19a4..3af7969 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideTimeWindowAggReduceGroupFunction.scala
@@ -25,9 +25,9 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* It is used for sliding windows on batch for time-windows. It takes a prepared input row (with
@@ -53,7 +53,8 @@ class DataSetSlideTimeWindowAggReduceGroupFunction(
extends RichGroupReduceFunction[Row, Row]
with CombineFunction[Row, Row]
with ResultTypeQueryable[Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private val timeFieldPos = returnType.getArity - 1
private val intermediateWindowStartPos = keysAndAggregatesArity
@@ -61,7 +62,6 @@ class DataSetSlideTimeWindowAggReduceGroupFunction(
protected var intermediateRow: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
index e4b9458..c64a522 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetSlideWindowAggReduceGroupFunction.scala
@@ -22,9 +22,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* It wraps the aggregate logic inside of
@@ -45,7 +45,8 @@ class DataSetSlideWindowAggReduceGroupFunction(
finalRowWindowEndPos: Option[Int],
windowSize: Long)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var collector: RowTimeWindowPropertyCollector = _
protected val windowStartPos: Int = keysAndAggregatesArity
@@ -53,7 +54,6 @@ class DataSetSlideWindowAggReduceGroupFunction(
private var output: Row = _
protected var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
protected var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
index 0e73f7b..22fe389 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleCountWindowAggReduceGroupFunction.scala
@@ -22,9 +22,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* It wraps the aggregate logic inside of
@@ -38,12 +38,12 @@ class DataSetTumbleCountWindowAggReduceGroupFunction(
private val genAggregations: GeneratedAggregationsFunction,
private val windowSize: Long)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var output: Row = _
private var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
index b4f7585..7ae4c17 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetTumbleTimeWindowAggReduceGroupFunction.scala
@@ -22,9 +22,9 @@ import java.lang.Iterable
import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* It wraps the aggregate logic inside of
@@ -44,7 +44,8 @@ class DataSetTumbleTimeWindowAggReduceGroupFunction(
windowEndPos: Option[Int],
keysAndAggregatesArity: Int)
extends RichGroupReduceFunction[Row, Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var collector: RowTimeWindowPropertyCollector = _
protected var aggregateBuffer: Row = new Row(keysAndAggregatesArity + 1)
@@ -52,7 +53,6 @@ class DataSetTumbleTimeWindowAggReduceGroupFunction(
private var output: Row = _
protected var accumulators: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
protected var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
index d49ed0e..324784f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/DataSetWindowAggMapFunction.scala
@@ -26,8 +26,8 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.LoggerFactory
/**
* This map function only works for windows on batch tables.
@@ -44,12 +44,12 @@ class DataSetWindowAggMapFunction(
@transient private val returnType: TypeInformation[Row])
extends RichMapFunction[Row, Row]
with ResultTypeQueryable[Row]
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var accs: Row = _
private var output: Row = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index a476987..df59460 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -19,16 +19,16 @@ package org.apache.flink.table.runtime.aggregate
import java.lang.{Long => JLong}
+import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.types.Row
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.state.{StateDescriptor, ValueState, ValueStateDescriptor}
-import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
-import org.slf4j.{Logger, LoggerFactory}
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
/**
* Aggregate Function used for the groupby (without window) aggregate
@@ -42,9 +42,9 @@ class GroupAggProcessFunction(
private val generateRetraction: Boolean,
private val queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
private var newRow: CRow = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 5c28519..1d947a0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -32,7 +32,7 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
/**
* Process Function used for the aggregate in bounded proc-time OVER window
@@ -50,13 +50,13 @@ class ProcTimeBoundedRangeOver(
inputType: TypeInformation[CRow],
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var output: CRow = _
private var accumulatorState: ValueState[Row] = _
private var rowMapState: MapState[Long, JList[Row]] = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index 09d98ad..ccddaa5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -18,22 +18,19 @@
package org.apache.flink.table.runtime.aggregate
import java.util
+import java.util.{List => JList}
+import org.apache.flink.api.common.state.{MapState, MapStateDescriptor, ValueState, ValueStateDescriptor}
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
+import org.apache.flink.api.java.typeutils.{ListTypeInfo, RowTypeInfo}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.types.Row
-import org.apache.flink.util.{Collector, Preconditions}
-import org.apache.flink.api.common.state._
-import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.ListTypeInfo
-import java.util.{List => JList}
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
+import org.apache.flink.types.Row
+import org.apache.flink.util.{Collector, Preconditions}
/**
* Process Function for ROW clause processing-time bounded OVER window
@@ -50,7 +47,8 @@ class ProcTimeBoundedRowsOver(
inputType: TypeInformation[CRow],
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
Preconditions.checkArgument(precedingOffset > 0)
@@ -60,7 +58,6 @@ class ProcTimeBoundedRowsOver(
private var counterState: ValueState[Long] = _
private var smallestTsState: ValueState[Long] = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
index 4fb5595..6e4c510 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
@@ -17,16 +17,16 @@
*/
package org.apache.flink.table.runtime.aggregate
+import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
+import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.types.Row
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.state.{StateDescriptor, ValueState, ValueStateDescriptor}
-import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.CRow
-import org.slf4j.LoggerFactory
+import org.apache.flink.table.util.Logging
+import org.apache.flink.types.Row
+import org.apache.flink.util.Collector
/**
* Process Function for processing-time unbounded OVER window
@@ -39,11 +39,11 @@ class ProcTimeUnboundedOver(
aggregationStateType: RowTypeInfo,
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
private var output: CRow = _
private var state: ValueState[Row] = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 1ee2693..85c523e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -28,9 +28,9 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.{Collector, Preconditions}
-import org.slf4j.LoggerFactory
/**
* Process Function for RANGE clause event-time bounded OVER window
@@ -48,7 +48,8 @@ class RowTimeBoundedRangeOver(
rowTimeIdx: Int,
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
Preconditions.checkNotNull(aggregationStateType)
Preconditions.checkNotNull(precedingOffset)
@@ -66,7 +67,6 @@ class RowTimeBoundedRangeOver(
// to this time stamp.
private var dataState: MapState[Long, JList[Row]] = _
- val LOG = LoggerFactory.getLogger(this.getClass)
private var function: GeneratedAggregations = _
override def open(config: Configuration) {
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index 60200bc..e120d6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -29,9 +29,9 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.{Collector, Preconditions}
-import org.slf4j.{Logger, LoggerFactory}
/**
* Process Function for ROWS clause event-time bounded OVER window
@@ -49,13 +49,12 @@ class RowTimeBoundedRowsOver(
rowTimeIdx: Int,
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
+ with Compiler[GeneratedAggregations]
+ with Logging {
Preconditions.checkNotNull(aggregationStateType)
Preconditions.checkNotNull(precedingOffset)
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
-
private var output: CRow = _
// the state which keeps the last triggering timestamp
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index c8183c9..c8236a3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -29,9 +29,9 @@ import org.apache.flink.streaming.api.operators.TimestampedCollector
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.{Logger, LoggerFactory}
/**
@@ -48,9 +48,8 @@ abstract class RowTimeUnboundedOver(
rowTimeIdx: Int,
queryConfig: StreamQueryConfig)
extends ProcessFunctionWithCleanupState[CRow, CRow](queryConfig)
- with Compiler[GeneratedAggregations] {
-
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ with Compiler[GeneratedAggregations]
+ with Logging {
protected var output: CRow = _
// state to hold the accumulators of the aggregations
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToJavaTupleMapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToJavaTupleMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToJavaTupleMapRunner.scala
index 95f304d..a9966f8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToJavaTupleMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToJavaTupleMapRunner.scala
@@ -27,8 +27,8 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.{Logger, LoggerFactory}
/**
* Convert [[CRow]] to a [[JTuple2]].
@@ -39,9 +39,8 @@ class CRowToJavaTupleMapRunner(
@transient var returnType: TypeInformation[JTuple2[JBool, Any]])
extends RichMapFunction[CRow, Any]
with ResultTypeQueryable[JTuple2[JBool, Any]]
- with Compiler[MapFunction[Row, Any]] {
-
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ with Compiler[MapFunction[Row, Any]]
+ with Logging {
private var function: MapFunction[Row, Any] = _
private var tupleWrapper: JTuple2[JBool, Any] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToScalaTupleMapRunner.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToScalaTupleMapRunner.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToScalaTupleMapRunner.scala
index c7d71a9..5b122f3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToScalaTupleMapRunner.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/conversion/CRowToScalaTupleMapRunner.scala
@@ -24,8 +24,8 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.{Logger, LoggerFactory}
/**
* Convert [[CRow]] to a [[Tuple2]].
@@ -36,9 +36,8 @@ class CRowToScalaTupleMapRunner(
@transient var returnType: TypeInformation[(Boolean, Any)])
extends RichMapFunction[CRow, (Boolean, Any)]
with ResultTypeQueryable[(Boolean, Any)]
- with Compiler[MapFunction[Row, Any]] {
-
- val LOG: Logger = LoggerFactory.getLogger(this.getClass)
+ with Compiler[MapFunction[Row, Any]]
+ with Logging {
private var function: MapFunction[Row, Any] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
index 1cb3a6e..fff5924 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/CRowValuesInputFormat.scala
@@ -24,8 +24,8 @@ import org.apache.flink.api.java.typeutils.ResultTypeQueryable
import org.apache.flink.core.io.GenericInputSplit
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.LoggerFactory
class CRowValuesInputFormat(
name: String,
@@ -34,9 +34,8 @@ class CRowValuesInputFormat(
extends GenericInputFormat[CRow]
with NonParallelInput
with ResultTypeQueryable[CRow]
- with Compiler[GenericInputFormat[Row]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[GenericInputFormat[Row]]
+ with Logging {
private var format: GenericInputFormat[Row] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
index 43ce605..858146a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/io/ValuesInputFormat.scala
@@ -21,10 +21,10 @@ package org.apache.flink.table.runtime.io
import org.apache.flink.api.common.io.{GenericInputFormat, NonParallelInput}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ResultTypeQueryable
-import org.apache.flink.table.codegen.Compiler
import org.apache.flink.core.io.GenericInputSplit
+import org.apache.flink.table.codegen.Compiler
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
-import org.slf4j.LoggerFactory
class ValuesInputFormat(
name: String,
@@ -33,9 +33,8 @@ class ValuesInputFormat(
extends GenericInputFormat[Row]
with NonParallelInput
with ResultTypeQueryable[Row]
- with Compiler[GenericInputFormat[Row]] {
-
- val LOG = LoggerFactory.getLogger(this.getClass)
+ with Compiler[GenericInputFormat[Row]]
+ with Logging {
private var format: GenericInputFormat[Row] = _
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/ProcTimeWindowInnerJoin.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/ProcTimeWindowInnerJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/ProcTimeWindowInnerJoin.scala
index e62a18f..8240376 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/ProcTimeWindowInnerJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/ProcTimeWindowInnerJoin.scala
@@ -30,9 +30,9 @@ import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.table.codegen.Compiler
import org.apache.flink.table.runtime.CRowWrappingCollector
import org.apache.flink.table.runtime.types.CRow
+import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.slf4j.LoggerFactory
/**
* A CoProcessFunction to support stream join stream, currently just support inner-join
@@ -55,7 +55,8 @@ class ProcTimeWindowInnerJoin(
private val genJoinFuncName: String,
private val genJoinFuncCode: String)
extends CoProcessFunction[CRow, CRow, CRow]
- with Compiler[FlatJoinFunction[Row, Row, Row]]{
+ with Compiler[FlatJoinFunction[Row, Row, Row]]
+ with Logging {
private var cRowWrapper: CRowWrappingCollector = _
@@ -80,8 +81,6 @@ class ProcTimeWindowInnerJoin(
private val leftStreamWinSize: Long = if (leftLowerBound <= 0) -leftLowerBound else -1
private val rightStreamWinSize: Long = if (leftUpperBound >= 0) leftUpperBound else -1
- val LOG = LoggerFactory.getLogger(this.getClass)
-
override def open(config: Configuration) {
LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n " +
s"Code:\n$genJoinFuncCode")
http://git-wip-us.apache.org/repos/asf/flink/blob/df7452d9/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/Logging.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/Logging.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/Logging.scala
new file mode 100644
index 0000000..b6be99e
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/Logging.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.util
+
+import org.slf4j.{Logger, LoggerFactory}
+
+/**
+ * Helper class to ensure the logger is never serialized.
+ */
+trait Logging {
+ @transient lazy val LOG: Logger = LoggerFactory.getLogger(getClass)
+}
[4/5] flink git commit: [FLINK-7206] [table] Add DataView to support
direct state access in AggregateFunction accumulators.
Posted by fh...@apache.org.
[FLINK-7206] [table] Add DataView to support direct state access in AggregateFunction accumulators.
This closes #4355.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1fc0b641
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1fc0b641
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1fc0b641
Branch: refs/heads/master
Commit: 1fc0b6413c74eff0ace25f4329451e35e84849b5
Parents: 88848e7
Author: 宝牛 <ba...@taobao.com>
Authored: Wed Aug 23 17:45:05 2017 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Aug 29 22:10:17 2017 +0200
----------------------------------------------------------------------
.../flink/table/api/dataview/DataView.scala | 35 +++
.../flink/table/api/dataview/DataViewSpec.scala | 55 +++++
.../flink/table/api/dataview/ListView.scala | 142 ++++++++++++
.../flink/table/api/dataview/MapView.scala | 198 ++++++++++++++++
.../codegen/AggregationCodeGenerator.scala | 231 ++++++++++++++++---
.../flink/table/codegen/CodeGenerator.scala | 27 ++-
.../table/dataview/ListViewSerializer.scala | 111 +++++++++
.../flink/table/dataview/ListViewTypeInfo.scala | 66 ++++++
.../dataview/ListViewTypeInfoFactory.scala | 43 ++++
.../table/dataview/MapViewSerializer.scala | 121 ++++++++++
.../flink/table/dataview/MapViewTypeInfo.scala | 72 ++++++
.../table/dataview/MapViewTypeInfoFactory.scala | 51 ++++
.../flink/table/dataview/StateListView.scala | 47 ++++
.../flink/table/dataview/StateMapView.scala | 54 +++++
.../utils/UserDefinedFunctionUtils.scala | 113 ++++++++-
.../table/runtime/aggregate/AggregateUtil.scala | 95 +++++---
.../aggregate/GeneratedAggregations.scala | 21 +-
.../aggregate/GroupAggProcessFunction.scala | 8 +-
.../aggregate/ProcTimeBoundedRangeOver.scala | 11 +-
.../aggregate/ProcTimeBoundedRowsOver.scala | 11 +-
.../aggregate/ProcTimeUnboundedOver.scala | 9 +-
.../aggregate/RowTimeBoundedRangeOver.scala | 6 +
.../aggregate/RowTimeBoundedRowsOver.scala | 6 +
.../aggregate/RowTimeUnboundedOver.scala | 5 +
.../utils/JavaUserDefinedAggFunctions.java | 199 ++++++++++++++++
.../table/dataview/ListViewSerializerTest.scala | 62 +++++
.../table/dataview/MapViewSerializerTest.scala | 68 ++++++
.../runtime/batch/table/AggregateITCase.scala | 9 +-
.../table/runtime/harness/HarnessTestBase.scala | 17 ++
.../runtime/stream/table/AggregateITCase.scala | 41 +++-
.../stream/table/GroupWindowITCase.scala | 40 ++--
.../runtime/stream/table/OverWindowITCase.scala | 118 +++++-----
32 files changed, 1933 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataView.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataView.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataView.scala
new file mode 100644
index 0000000..2214086
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataView.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.dataview
+
+/**
+ * A [[DataView]] is a collection type that can be used in the accumulator of an
+ * [[org.apache.flink.table.functions.AggregateFunction]].
+ *
+ * Depending on the context in which the [[org.apache.flink.table.functions.AggregateFunction]] is
+ * used, a [[DataView]] can be backed by a Java heap collection or a state backend.
+ */
+trait DataView extends Serializable {
+
+ /**
+ * Clears the [[DataView]] and removes all data.
+ */
+ def clear(): Unit
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataViewSpec.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataViewSpec.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataViewSpec.scala
new file mode 100644
index 0000000..943fe03
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/DataViewSpec.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.dataview
+
+import java.lang.reflect.Field
+
+import org.apache.flink.api.common.state.{ListStateDescriptor, MapStateDescriptor, State, StateDescriptor}
+import org.apache.flink.table.dataview.{ListViewTypeInfo, MapViewTypeInfo}
+
+/**
+ * Data view specification.
+ *
+ * @tparam ACC type extends [[DataView]]
+ */
+trait DataViewSpec[ACC <: DataView] {
+ def stateId: String
+ def field: Field
+ def toStateDescriptor: StateDescriptor[_ <: State, _]
+}
+
+case class ListViewSpec[T](
+ stateId: String,
+ field: Field,
+ listViewTypeInfo: ListViewTypeInfo[T])
+ extends DataViewSpec[ListView[T]] {
+
+ override def toStateDescriptor: StateDescriptor[_ <: State, _] =
+ new ListStateDescriptor[T](stateId, listViewTypeInfo.elementType)
+}
+
+case class MapViewSpec[K, V](
+ stateId: String,
+ field: Field,
+ mapViewTypeInfo: MapViewTypeInfo[K, V])
+ extends DataViewSpec[MapView[K, V]] {
+
+ override def toStateDescriptor: StateDescriptor[_ <: State, _] =
+ new MapStateDescriptor[K, V](stateId, mapViewTypeInfo.keyType, mapViewTypeInfo.valueType)
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/ListView.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/ListView.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/ListView.scala
new file mode 100644
index 0000000..59b2426
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/ListView.scala
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.dataview
+
+import java.lang.{Iterable => JIterable}
+import java.util
+
+import org.apache.flink.api.common.typeinfo.{TypeInfo, TypeInformation}
+import org.apache.flink.table.dataview.ListViewTypeInfoFactory
+
+/**
+ * A [[ListView]] provides List functionality for accumulators used by user-defined aggregate
+ * functions [[org.apache.flink.api.common.functions.AggregateFunction]].
+ *
+ * A [[ListView]] can be backed by a Java ArrayList or a state backend, depending on the context in
+ * which the aggregate function is used.
+ *
+ * At runtime [[ListView]] will be replaced by a [[org.apache.flink.table.dataview.StateListView]]
+ * if it is backed by a state backend.
+ *
+ * Example of an accumulator type with a [[ListView]] and an aggregate function that uses it:
+ * {{{
+ *
+ * public class MyAccum {
+ * public ListView<String> list;
+ * public long count;
+ * }
+ *
+ * public class MyAgg extends AggregateFunction<Long, MyAccum> {
+ *
+ * @Override
+ * public MyAccum createAccumulator() {
+ * MyAccum accum = new MyAccum();
+ * accum.list = new ListView<>(Types.STRING);
+ * accum.count = 0L;
+ * return accum;
+ * }
+ *
+ * public void accumulate(MyAccum accumulator, String id) {
+ * accumulator.list.add(id);
+ * ... ...
+ * accumulator.get()
+ * ... ...
+ * }
+ *
+ * @Override
+ * public Long getValue(MyAccum accumulator) {
+ * accumulator.list.add(id);
+ * ... ...
+ * accumulator.get()
+ * ... ...
+ * return accumulator.count;
+ * }
+ * }
+ *
+ * }}}
+ *
+ * @param elementTypeInfo element type information
+ * @tparam T element type
+ */
+@TypeInfo(classOf[ListViewTypeInfoFactory[_]])
+class ListView[T](
+ @transient private[flink] val elementTypeInfo: TypeInformation[T],
+ private[flink] val list: util.List[T])
+ extends DataView {
+
+ /**
+ * Creates a list view for elements of the specified type.
+ *
+ * @param elementTypeInfo The type of the list view elements.
+ */
+ def this(elementTypeInfo: TypeInformation[T]) {
+ this(elementTypeInfo, new util.ArrayList[T]())
+ }
+
+ /**
+ * Creates a list view.
+ */
+ def this() = this(null)
+
+ /**
+ * Returns an iterable of the list view.
+ *
+ * @throws Exception Thrown if the system cannot get data.
+ * @return The iterable of the list or { @code null} if the list is empty.
+ */
+ @throws[Exception]
+ def get: JIterable[T] = {
+ if (!list.isEmpty) {
+ list
+ } else {
+ null
+ }
+ }
+
+ /**
+ * Adds the given value to the list.
+ *
+ * @throws Exception Thrown if the system cannot add data.
+ * @param value The element to be appended to this list view.
+ */
+ @throws[Exception]
+ def add(value: T): Unit = list.add(value)
+
+ /**
+ * Adds all of the elements of the specified list to this list view.
+ *
+ * @throws Exception Thrown if the system cannot add all data.
+ * @param list The list with the elements that will be stored in this list view.
+ */
+ @throws[Exception]
+ def addAll(list: util.List[T]): Unit = this.list.addAll(list)
+
+ /**
+ * Removes all of the elements from this list view.
+ */
+ override def clear(): Unit = list.clear()
+
+ override def equals(other: Any): Boolean = other match {
+ case that: ListView[T] =>
+ list.equals(that.list)
+ case _ => false
+ }
+
+ override def hashCode(): Int = list.hashCode()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/MapView.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/MapView.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/MapView.scala
new file mode 100644
index 0000000..9206d6a
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/dataview/MapView.scala
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api.dataview
+
+import java.lang.{Iterable => JIterable}
+import java.util
+
+import org.apache.flink.api.common.typeinfo.{TypeInfo, TypeInformation}
+import org.apache.flink.table.dataview.MapViewTypeInfoFactory
+
+/**
+ * A [[MapView]] provides Map functionality for accumulators used by user-defined aggregate
+ * functions [[org.apache.flink.table.functions.AggregateFunction]].
+ *
+ * A [[MapView]] can be backed by a Java HashMap or a state backend, depending on the context in
+ * which the aggregation function is used.
+ *
+ * At runtime [[MapView]] will be replaced by a [[org.apache.flink.table.dataview.StateMapView]]
+ * if it is backed by a state backend.
+ *
+ * Example of an accumulator type with a [[MapView]] and an aggregate function that uses it:
+ * {{{
+ *
+ * public class MyAccum {
+ * public MapView<String, Integer> map;
+ * public long count;
+ * }
+ *
+ * public class MyAgg extends AggregateFunction<Long, MyAccum> {
+ *
+ * @Override
+ * public MyAccum createAccumulator() {
+ * MyAccum accum = new MyAccum();
+ * accum.map = new MapView<>(Types.STRING, Types.INT);
+ * accum.count = 0L;
+ * return accum;
+ * }
+ *
+ * public void accumulate(MyAccum accumulator, String id) {
+ * try {
+ * if (!accumulator.map.contains(id)) {
+ * accumulator.map.put(id, 1);
+ * accumulator.count++;
+ * }
+ * } catch (Exception e) {
+ * e.printStackTrace();
+ * }
+ * }
+ *
+ * @Override
+ * public Long getValue(MyAccum accumulator) {
+ * return accumulator.count;
+ * }
+ * }
+ *
+ * }}}
+ *
+ * @param keyTypeInfo key type information
+ * @param valueTypeInfo value type information
+ * @tparam K key type
+ * @tparam V value type
+ */
+@TypeInfo(classOf[MapViewTypeInfoFactory[_, _]])
+class MapView[K, V](
+ @transient private[flink] val keyTypeInfo: TypeInformation[K],
+ @transient private[flink] val valueTypeInfo: TypeInformation[V],
+ private[flink] val map: util.Map[K, V])
+ extends DataView {
+
+ /**
+ * Creates a MapView with the specified key and value types.
+ *
+ * @param keyTypeInfo The type of keys of the MapView.
+ * @param valueTypeInfo The type of the values of the MapView.
+ */
+ def this(keyTypeInfo: TypeInformation[K], valueTypeInfo: TypeInformation[V]) {
+ this(keyTypeInfo, valueTypeInfo, new util.HashMap[K, V]())
+ }
+
+ /**
+ * Creates a MapView.
+ */
+ def this() = this(null, null)
+
+ /**
+ * Return the value for the specified key or { @code null } if the key is not in the map view.
+ *
+ * @param key The look up key.
+ * @return The value for the specified key.
+ * @throws Exception Thrown if the system cannot get data.
+ */
+ @throws[Exception]
+ def get(key: K): V = map.get(key)
+
+ /**
+ * Inserts a value for the given key into the map view.
+ * If the map view already contains a value for the key, the existing value is overwritten.
+ *
+ * @param key The key for which the value is inserted.
+ * @param value The value that is inserted for the key.
+ * @throws Exception Thrown if the system cannot put data.
+ */
+ @throws[Exception]
+ def put(key: K, value: V): Unit = map.put(key, value)
+
+ /**
+ * Inserts all mappings from the specified map to this map view.
+ *
+ * @param map The map whose entries are inserted into this map view.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def putAll(map: util.Map[K, V]): Unit = this.map.putAll(map)
+
+ /**
+ * Deletes the value for the given key.
+ *
+ * @param key The key for which the value is deleted.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def remove(key: K): Unit = map.remove(key)
+
+ /**
+ * Checks if the map view contains a value for a given key.
+ *
+ * @param key The key to check.
+ * @return True if there exists a value for the given key, false otherwise.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def contains(key: K): Boolean = map.containsKey(key)
+
+ /**
+ * Returns all entries of the map view.
+ *
+ * @return An iterable of all the key-value pairs in the map view.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def entries: JIterable[util.Map.Entry[K, V]] = map.entrySet()
+
+ /**
+ * Returns all the keys in the map view.
+ *
+ * @return An iterable of all the keys in the map.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def keys: JIterable[K] = map.keySet()
+
+ /**
+ * Returns all the values in the map view.
+ *
+ * @return An iterable of all the values in the map.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def values: JIterable[V] = map.values()
+
+ /**
+ * Returns an iterator over all entries of the map view.
+ *
+ * @return An iterator over all the mappings in the map.
+ * @throws Exception Thrown if the system cannot access the map.
+ */
+ @throws[Exception]
+ def iterator: util.Iterator[util.Map.Entry[K, V]] = map.entrySet().iterator()
+
+ /**
+ * Removes all entries of this map.
+ */
+ override def clear(): Unit = map.clear()
+
+ override def equals(other: Any): Boolean = other match {
+ case that: MapView[K, V] =>
+ map.equals(that.map)
+ case _ => false
+ }
+
+ override def hashCode(): Int = map.hashCode()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
index 25527cc..22ce5ba 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/AggregationCodeGenerator.scala
@@ -17,17 +17,23 @@
*/
package org.apache.flink.table.codegen
-import java.lang.reflect.ParameterizedType
+import java.lang.reflect.{Modifier, ParameterizedType}
import java.lang.{Iterable => JIterable}
+import org.apache.commons.codec.binary.Base64
+import org.apache.flink.api.common.state.{State, StateDescriptor}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.TableConfig
+import org.apache.flink.table.api.dataview._
import org.apache.flink.table.codegen.Indenter.toISC
-import org.apache.flink.table.codegen.CodeGenUtils.newName
+import org.apache.flink.table.codegen.CodeGenUtils.{newName, reflectiveFieldWriteAccess}
import org.apache.flink.table.functions.AggregateFunction
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils.{getUserDefinedMethod, signatureToString}
import org.apache.flink.table.runtime.aggregate.{GeneratedAggregations, SingleElementIterable}
+import org.apache.flink.util.InstantiationUtil
+
+import scala.collection.mutable
/**
* A code generator for generating [[GeneratedAggregations]].
@@ -42,6 +48,18 @@ class AggregationCodeGenerator(
input: TypeInformation[_ <: Any])
extends CodeGenerator(config, nullableInput, input) {
+ // set of statements for cleanup dataview that will be added only once
+ // we use a LinkedHashSet to keep the insertion order
+ private val reusableCleanupStatements = mutable.LinkedHashSet[String]()
+
+ /**
+ * @return code block of statements that need to be placed in the cleanup() method of
+ * [[GeneratedAggregations]]
+ */
+ def reuseCleanupCode(): String = {
+ reusableCleanupStatements.mkString("", "\n", "\n")
+ }
+
/**
* Generates a [[org.apache.flink.table.runtime.aggregate.GeneratedAggregations]] that can be
* passed to a Java compiler.
@@ -79,13 +97,15 @@ class AggregationCodeGenerator(
outputArity: Int,
needRetract: Boolean,
needMerge: Boolean,
- needReset: Boolean)
+ needReset: Boolean,
+ accConfig: Option[Array[Seq[DataViewSpec[_]]]])
: GeneratedAggregationsFunction = {
// get unique function name
val funcName = newName(name)
// register UDAGGs
- val aggs = aggregates.map(a => addReusableFunction(a))
+ val aggs = aggregates.map(a => addReusableFunction(a, contextTerm))
+
// get java types of accumulators
val accTypeClasses = aggregates.map { a =>
a.getClass.getMethod("createAccumulator").getReturnType
@@ -105,6 +125,9 @@ class AggregationCodeGenerator(
inFields.map(classes(_))
}
+ // initialize and create data views
+ addReusableDataViews()
+
// check and validate the needed methods
aggregates.zipWithIndex.map {
case (a, i) =>
@@ -161,13 +184,121 @@ class AggregationCodeGenerator(
}
}
+ /**
+ * Create DataView Term, for example, acc1_map_dataview.
+ *
+ * @param aggIndex index of aggregate function
+ * @param fieldName field name of DataView
+ * @return term to access [[MapView]] or [[ListView]]
+ */
+ def createDataViewTerm(aggIndex: Int, fieldName: String): String = {
+ s"acc${aggIndex}_${fieldName}_dataview"
+ }
+
+ /**
+ * Adds a reusable [[org.apache.flink.table.api.dataview.DataView]] to the open, cleanup,
+ * close and member area of the generated function.
+ *
+ */
+ def addReusableDataViews(): Unit = {
+ if (accConfig.isDefined) {
+ val descMapping: Map[String, StateDescriptor[_, _]] = accConfig.get
+ .flatMap(specs => specs.map(s => (s.stateId, s.toStateDescriptor)))
+ .toMap[String, StateDescriptor[_ <: State, _]]
+
+ for (i <- aggs.indices) yield {
+ for (spec <- accConfig.get(i)) yield {
+ val dataViewField = spec.field
+ val dataViewTypeTerm = dataViewField.getType.getCanonicalName
+ val desc = descMapping.getOrElse(spec.stateId,
+ throw new CodeGenException(
+ s"Can not find DataView in accumulator by id: ${spec.stateId}"))
+
+ // define the DataView variables
+ val serializedData = serializeStateDescriptor(desc)
+ val dataViewFieldTerm = createDataViewTerm(i, dataViewField.getName)
+ val field =
+ s"""
+ | transient $dataViewTypeTerm $dataViewFieldTerm = null;
+ |""".stripMargin
+ reusableMemberStatements.add(field)
+
+ // create DataViews
+ val descFieldTerm = s"${dataViewFieldTerm}_desc"
+ val descClassQualifier = classOf[StateDescriptor[_, _]].getCanonicalName
+ val descDeserializeCode =
+ s"""
+ | $descClassQualifier $descFieldTerm = ($descClassQualifier)
+ | org.apache.flink.util.InstantiationUtil.deserializeObject(
+ | org.apache.commons.codec.binary.Base64.decodeBase64("$serializedData"),
+ | $contextTerm.getUserCodeClassLoader());
+ |""".stripMargin
+ val createDataView = if (dataViewField.getType == classOf[MapView[_, _]]) {
+ s"""
+ | $descDeserializeCode
+ | $dataViewFieldTerm = new org.apache.flink.table.dataview.StateMapView(
+ | $contextTerm.getMapState((
+ | org.apache.flink.api.common.state.MapStateDescriptor)$descFieldTerm));
+ |""".stripMargin
+ } else if (dataViewField.getType == classOf[ListView[_]]) {
+ s"""
+ | $descDeserializeCode
+ | $dataViewFieldTerm = new org.apache.flink.table.dataview.StateListView(
+ | $contextTerm.getListState((
+ | org.apache.flink.api.common.state.ListStateDescriptor)$descFieldTerm));
+ |""".stripMargin
+ } else {
+ throw new CodeGenException(s"Unsupported dataview type: $dataViewTypeTerm")
+ }
+ reusableOpenStatements.add(createDataView)
+
+ // cleanup DataViews
+ val cleanup =
+ s"""
+ | $dataViewFieldTerm.clear();
+ |""".stripMargin
+ reusableCleanupStatements.add(cleanup)
+ }
+ }
+ }
+ }
+
+ /**
+ * Generate statements to set data view field when use state backend.
+ *
+ * @param accTerm aggregation term
+ * @param aggIndex index of aggregation
+ * @return data view field set statements
+ */
+ def genDataViewFieldSetter(accTerm: String, aggIndex: Int): String = {
+ if (accConfig.isDefined) {
+ val setters = for (spec <- accConfig.get(aggIndex)) yield {
+ val field = spec.field
+ val dataViewTerm = createDataViewTerm(aggIndex, field.getName)
+ val fieldSetter = if (Modifier.isPublic(field.getModifiers)) {
+ s"$accTerm.${field.getName} = $dataViewTerm;"
+ } else {
+ val fieldTerm = addReusablePrivateFieldAccess(field.getDeclaringClass, field.getName)
+ s"${reflectiveFieldWriteAccess(fieldTerm, field, accTerm, dataViewTerm)};"
+ }
+
+ s"""
+ | $fieldSetter
+ """.stripMargin
+ }
+ setters.mkString("\n")
+ } else {
+ ""
+ }
+ }
+
def genSetAggregationResults: String = {
val sig: String =
j"""
| public final void setAggregationResults(
| org.apache.flink.types.Row accs,
- | org.apache.flink.types.Row output)""".stripMargin
+ | org.apache.flink.types.Row output) throws Exception """.stripMargin
val setAggs: String = {
for (i <- aggs.indices) yield
@@ -181,10 +312,11 @@ class AggregationCodeGenerator(
j"""
| org.apache.flink.table.functions.AggregateFunction baseClass$i =
| (org.apache.flink.table.functions.AggregateFunction) ${aggs(i)};
- |
+ | ${accTypes(i)} acc$i = (${accTypes(i)}) accs.getField($i);
+ | ${genDataViewFieldSetter(s"acc$i", i)}
| output.setField(
| ${aggMapping(i)},
- | baseClass$i.getValue((${accTypes(i)}) accs.getField($i)));""".stripMargin
+ | baseClass$i.getValue(acc$i));""".stripMargin
}
}.mkString("\n")
@@ -200,14 +332,17 @@ class AggregationCodeGenerator(
j"""
| public final void accumulate(
| org.apache.flink.types.Row accs,
- | org.apache.flink.types.Row input)""".stripMargin
+ | org.apache.flink.types.Row input) throws Exception """.stripMargin
val accumulate: String = {
- for (i <- aggs.indices) yield
+ for (i <- aggs.indices) yield {
j"""
+ | ${accTypes(i)} acc$i = (${accTypes(i)}) accs.getField($i);
+ | ${genDataViewFieldSetter(s"acc$i", i)}
| ${aggs(i)}.accumulate(
- | ((${accTypes(i)}) accs.getField($i)),
+ | acc$i,
| ${parametersCode(i)});""".stripMargin
+ }
}.mkString("\n")
j"""$sig {
@@ -221,14 +356,17 @@ class AggregationCodeGenerator(
j"""
| public final void retract(
| org.apache.flink.types.Row accs,
- | org.apache.flink.types.Row input)""".stripMargin
+ | org.apache.flink.types.Row input) throws Exception """.stripMargin
val retract: String = {
- for (i <- aggs.indices) yield
+ for (i <- aggs.indices) yield {
j"""
+ | ${accTypes(i)} acc$i = (${accTypes(i)}) accs.getField($i);
+ | ${genDataViewFieldSetter(s"acc$i", i)}
| ${aggs(i)}.retract(
- | ((${accTypes(i)}) accs.getField($i)),
+ | acc$i,
| ${parametersCode(i)});""".stripMargin
+ }
}.mkString("\n")
if (needRetract) {
@@ -247,7 +385,7 @@ class AggregationCodeGenerator(
val sig: String =
j"""
- | public final org.apache.flink.types.Row createAccumulators()
+ | public final org.apache.flink.types.Row createAccumulators() throws Exception
| """.stripMargin
val init: String =
j"""
@@ -255,12 +393,15 @@ class AggregationCodeGenerator(
| new org.apache.flink.types.Row(${aggs.length});"""
.stripMargin
val create: String = {
- for (i <- aggs.indices) yield
+ for (i <- aggs.indices) yield {
j"""
+ | ${accTypes(i)} acc$i = (${accTypes(i)}) ${aggs(i)}.createAccumulator();
+ | ${genDataViewFieldSetter(s"acc$i", i)}
| accs.setField(
| $i,
- | ${aggs(i)}.createAccumulator());"""
- .stripMargin
+ | acc$i);"""
+ .stripMargin
+ }
}.mkString("\n")
val ret: String =
j"""
@@ -356,6 +497,10 @@ class AggregationCodeGenerator(
""".stripMargin
if (needMerge) {
+ if (accConfig.isDefined) {
+ throw new CodeGenException("DataView doesn't support merge when the backend uses " +
+ s"state when generate aggregation for $funcName.")
+ }
j"""
|$sig {
|$merge
@@ -385,13 +530,15 @@ class AggregationCodeGenerator(
val sig: String =
j"""
| public final void resetAccumulator(
- | org.apache.flink.types.Row accs)""".stripMargin
+ | org.apache.flink.types.Row accs) throws Exception """.stripMargin
val reset: String = {
- for (i <- aggs.indices) yield
+ for (i <- aggs.indices) yield {
j"""
- | ${aggs(i)}.resetAccumulator(
- | ((${accTypes(i)}) accs.getField($i)));""".stripMargin
+ | ${accTypes(i)} acc$i = (${accTypes(i)}) accs.getField($i);
+ | ${genDataViewFieldSetter(s"acc$i", i)}
+ | ${aggs(i)}.resetAccumulator(acc$i);""".stripMargin
+ }
}.mkString("\n")
if (needReset) {
@@ -404,6 +551,17 @@ class AggregationCodeGenerator(
}
}
+ val aggFuncCode = Seq(
+ genSetAggregationResults,
+ genAccumulate,
+ genRetract,
+ genCreateAccumulators,
+ genSetForwardedFields,
+ genSetConstantFlags,
+ genCreateOutputRow,
+ genMergeAccumulatorsPair,
+ genResetAccumulator).mkString("\n")
+
val generatedAggregationsClass = classOf[GeneratedAggregations].getCanonicalName
var funcCode =
j"""
@@ -416,20 +574,29 @@ class AggregationCodeGenerator(
| }
| ${reuseConstructorCode(funcName)}
|
+ | public final void open(
+ | org.apache.flink.api.common.functions.RuntimeContext $contextTerm) throws Exception {
+ | ${reuseOpenCode()}
+ | }
+ |
+ | $aggFuncCode
+ |
+ | public final void cleanup() throws Exception {
+ | ${reuseCleanupCode()}
+ | }
+ |
+ | public final void close() throws Exception {
+ | ${reuseCloseCode()}
+ | }
+ |}
""".stripMargin
- funcCode += genSetAggregationResults + "\n"
- funcCode += genAccumulate + "\n"
- funcCode += genRetract + "\n"
- funcCode += genCreateAccumulators + "\n"
- funcCode += genSetForwardedFields + "\n"
- funcCode += genSetConstantFlags + "\n"
- funcCode += genCreateOutputRow + "\n"
- funcCode += genMergeAccumulatorsPair + "\n"
- funcCode += genResetAccumulator + "\n"
- funcCode += "}"
-
GeneratedAggregationsFunction(funcName, funcCode)
}
+ @throws[Exception]
+ def serializeStateDescriptor(stateDescriptor: StateDescriptor[_, _]): String = {
+ val byteArray = InstantiationUtil.serializeObject(stateDescriptor)
+ Base64.encodeBase64URLSafeString(byteArray)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 946c6cd..154e8ad 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -41,8 +41,9 @@ import org.apache.flink.table.codegen.calls.FunctionGenerator
import org.apache.flink.table.codegen.calls.ScalarOperators._
import org.apache.flink.table.functions.sql.{ProctimeSqlFunction, ScalarSqlFunctions}
import org.apache.flink.table.functions.utils.UserDefinedFunctionUtils
-import org.apache.flink.table.functions.{FunctionContext, UserDefinedFunction}
+
import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo
+import org.apache.flink.table.functions.{FunctionContext, UserDefinedFunction}
import org.apache.flink.table.typeutils.TypeCheckUtils._
import scala.collection.JavaConversions._
@@ -108,31 +109,31 @@ abstract class CodeGenerator(
// set of member statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- private val reusableMemberStatements = mutable.LinkedHashSet[String]()
+ protected val reusableMemberStatements = mutable.LinkedHashSet[String]()
// set of constructor statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- private val reusableInitStatements = mutable.LinkedHashSet[String]()
+ protected val reusableInitStatements = mutable.LinkedHashSet[String]()
// set of open statements for RichFunction that will be added only once
// we use a LinkedHashSet to keep the insertion order
- private val reusableOpenStatements = mutable.LinkedHashSet[String]()
+ protected val reusableOpenStatements = mutable.LinkedHashSet[String]()
// set of close statements for RichFunction that will be added only once
// we use a LinkedHashSet to keep the insertion order
- private val reusableCloseStatements = mutable.LinkedHashSet[String]()
+ protected val reusableCloseStatements = mutable.LinkedHashSet[String]()
// set of statements that will be added only once per record
// we use a LinkedHashSet to keep the insertion order
- private val reusablePerRecordStatements = mutable.LinkedHashSet[String]()
+ protected val reusablePerRecordStatements = mutable.LinkedHashSet[String]()
// map of initial input unboxing expressions that will be added only once
// (inputTerm, index) -> expr
- private val reusableInputUnboxingExprs = mutable.Map[(String, Int), GeneratedExpression]()
+ protected val reusableInputUnboxingExprs = mutable.Map[(String, Int), GeneratedExpression]()
// set of constructor statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- private val reusableConstructorStatements = mutable.LinkedHashSet[(String, String)]()
+ protected val reusableConstructorStatements = mutable.LinkedHashSet[(String, String)]()
/**
* @return code block of statements that need to be placed in the member area of the Function
@@ -1458,9 +1459,10 @@ abstract class CodeGenerator(
* Adds a reusable [[UserDefinedFunction]] to the member area of the generated [[Function]].
*
* @param function [[UserDefinedFunction]] object to be instantiated during runtime
+ * @param contextTerm [[RuntimeContext]] term to access the [[RuntimeContext]]
* @return member variable term
*/
- def addReusableFunction(function: UserDefinedFunction): String = {
+ def addReusableFunction(function: UserDefinedFunction, contextTerm: String = null): String = {
val classQualifier = function.getClass.getCanonicalName
val functionSerializedData = UserDefinedFunctionUtils.serialize(function)
val fieldTerm = s"function_${function.functionIdentifier}"
@@ -1480,10 +1482,15 @@ abstract class CodeGenerator(
reusableInitStatements.add(functionDeserialization)
- val openFunction =
+ val openFunction = if (contextTerm != null) {
+ s"""
+ |$fieldTerm.open(new ${classOf[FunctionContext].getCanonicalName}($contextTerm));
+ """.stripMargin
+ } else {
s"""
|$fieldTerm.open(new ${classOf[FunctionContext].getCanonicalName}(getRuntimeContext()));
""".stripMargin
+ }
reusableOpenStatements.add(openFunction)
val closeFunction =
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala
new file mode 100644
index 0000000..a450c2c
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewSerializer.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import org.apache.flink.api.common.typeutils._
+import org.apache.flink.api.common.typeutils.base.{CollectionSerializerConfigSnapshot, ListSerializer}
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
+import org.apache.flink.table.api.dataview.ListView
+
+/**
+ * A serializer for [[ListView]]. The serializer relies on an element
+ * serializer for the serialization of the list's elements.
+ *
+ * The serialization format for the list is as follows: four bytes for the length of the list,
+ * followed by the serialized representation of each element.
+ *
+ * @param listSerializer List serializer.
+ * @tparam T The type of element in the list.
+ */
+class ListViewSerializer[T](val listSerializer: ListSerializer[T])
+ extends TypeSerializer[ListView[T]] {
+
+ override def isImmutableType: Boolean = false
+
+ override def duplicate(): TypeSerializer[ListView[T]] = {
+ new ListViewSerializer[T](listSerializer.duplicate().asInstanceOf[ListSerializer[T]])
+ }
+
+ override def createInstance(): ListView[T] = {
+ new ListView[T]
+ }
+
+ override def copy(from: ListView[T]): ListView[T] = {
+ new ListView[T](null, listSerializer.copy(from.list))
+ }
+
+ override def copy(from: ListView[T], reuse: ListView[T]): ListView[T] = copy(from)
+
+ override def getLength: Int = -1
+
+ override def serialize(record: ListView[T], target: DataOutputView): Unit = {
+ listSerializer.serialize(record.list, target)
+ }
+
+ override def deserialize(source: DataInputView): ListView[T] = {
+ new ListView[T](null, listSerializer.deserialize(source))
+ }
+
+ override def deserialize(reuse: ListView[T], source: DataInputView): ListView[T] =
+ deserialize(source)
+
+ override def copy(source: DataInputView, target: DataOutputView): Unit =
+ listSerializer.copy(source, target)
+
+ override def canEqual(obj: scala.Any): Boolean = obj != null && obj.getClass == getClass
+
+ override def hashCode(): Int = listSerializer.hashCode()
+
+ override def equals(obj: Any): Boolean = canEqual(this) &&
+ listSerializer.equals(obj.asInstanceOf[ListViewSerializer[_]].listSerializer)
+
+ override def snapshotConfiguration(): TypeSerializerConfigSnapshot =
+ listSerializer.snapshotConfiguration()
+
+ // copy and modified from ListSerializer.ensureCompatibility
+ override def ensureCompatibility(
+ configSnapshot: TypeSerializerConfigSnapshot): CompatibilityResult[ListView[T]] = {
+
+ configSnapshot match {
+ case snapshot: CollectionSerializerConfigSnapshot[_] =>
+ val previousListSerializerAndConfig = snapshot.getSingleNestedSerializerAndConfig
+
+ val compatResult = CompatibilityUtil.resolveCompatibilityResult(
+ previousListSerializerAndConfig.f0,
+ classOf[UnloadableDummyTypeSerializer[_]],
+ previousListSerializerAndConfig.f1,
+ listSerializer.getElementSerializer)
+
+ if (!compatResult.isRequiresMigration) {
+ CompatibilityResult.compatible[ListView[T]]
+ } else if (compatResult.getConvertDeserializer != null) {
+ CompatibilityResult.requiresMigration(
+ new ListViewSerializer[T](
+ new ListSerializer[T](
+ new TypeDeserializerAdapter[T](compatResult.getConvertDeserializer))
+ )
+ )
+ } else {
+ CompatibilityResult.requiresMigration[ListView[T]]
+ }
+
+ case _ => CompatibilityResult.requiresMigration[ListView[T]]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfo.scala
new file mode 100644
index 0000000..a10b675
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfo.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils.base.ListSerializer
+import org.apache.flink.table.api.dataview.ListView
+
+/**
+ * [[ListView]] type information.
+ *
+ * @param elementType element type information
+ * @tparam T element type
+ */
+class ListViewTypeInfo[T](val elementType: TypeInformation[T])
+ extends TypeInformation[ListView[T]] {
+
+ override def isBasicType: Boolean = false
+
+ override def isTupleType: Boolean = false
+
+ override def getArity: Int = 1
+
+ override def getTotalFields: Int = 1
+
+ override def getTypeClass: Class[ListView[T]] = classOf[ListView[T]]
+
+ override def isKeyType: Boolean = false
+
+ override def createSerializer(config: ExecutionConfig): TypeSerializer[ListView[T]] = {
+ val typeSer = elementType.createSerializer(config)
+ new ListViewSerializer[T](new ListSerializer[T](typeSer))
+ }
+
+ override def canEqual(obj: scala.Any): Boolean = obj != null && obj.getClass == getClass
+
+ override def hashCode(): Int = 31 * elementType.hashCode
+
+ override def equals(obj: Any): Boolean = canEqual(obj) && {
+ obj match {
+ case other: ListViewTypeInfo[T] =>
+ elementType.equals(other.elementType)
+ case _ => false
+ }
+ }
+
+ override def toString: String = s"ListView<$elementType>"
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfoFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfoFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfoFactory.scala
new file mode 100644
index 0000000..eda6cb9
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/ListViewTypeInfoFactory.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import java.lang.reflect.Type
+import java.util
+
+import org.apache.flink.api.common.typeinfo.{TypeInfoFactory, TypeInformation}
+import org.apache.flink.api.java.typeutils.GenericTypeInfo
+import org.apache.flink.table.api.dataview.ListView
+
+class ListViewTypeInfoFactory[T] extends TypeInfoFactory[ListView[T]] {
+
+ override def createTypeInfo(
+ t: Type,
+ genericParameters: util.Map[String, TypeInformation[_]]): TypeInformation[ListView[T]] = {
+
+ var elementType = genericParameters.get("T")
+
+ if (elementType == null) {
+ // we might can get the elementType later from the ListView constructor
+ elementType = new GenericTypeInfo(classOf[Any])
+ }
+
+ new ListViewTypeInfo[T](elementType.asInstanceOf[TypeInformation[T]])
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala
new file mode 100644
index 0000000..c53f10c
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewSerializer.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import org.apache.flink.api.common.typeutils._
+import org.apache.flink.api.common.typeutils.base.{MapSerializer, MapSerializerConfigSnapshot}
+import org.apache.flink.core.memory.{DataInputView, DataOutputView}
+import org.apache.flink.table.api.dataview.MapView
+
+/**
+ * A serializer for [[MapView]]. The serializer relies on a key serializer and a value
+ * serializer for the serialization of the map's key-value pairs.
+ *
+ * The serialization format for the map is as follows: four bytes for the length of the map,
+ * followed by the serialized representation of each key-value pair. To allow null values,
+ * each value is prefixed by a null marker.
+ *
+ * @param mapSerializer Map serializer.
+ * @tparam K The type of the keys in the map.
+ * @tparam V The type of the values in the map.
+ */
+class MapViewSerializer[K, V](val mapSerializer: MapSerializer[K, V])
+ extends TypeSerializer[MapView[K, V]] {
+
+ override def isImmutableType: Boolean = false
+
+ override def duplicate(): TypeSerializer[MapView[K, V]] =
+ new MapViewSerializer[K, V](
+ mapSerializer.duplicate().asInstanceOf[MapSerializer[K, V]])
+
+ override def createInstance(): MapView[K, V] = {
+ new MapView[K, V]()
+ }
+
+ override def copy(from: MapView[K, V]): MapView[K, V] = {
+ new MapView[K, V](null, null, mapSerializer.copy(from.map))
+ }
+
+ override def copy(from: MapView[K, V], reuse: MapView[K, V]): MapView[K, V] = copy(from)
+
+ override def getLength: Int = -1 // var length
+
+ override def serialize(record: MapView[K, V], target: DataOutputView): Unit = {
+ mapSerializer.serialize(record.map, target)
+ }
+
+ override def deserialize(source: DataInputView): MapView[K, V] = {
+ new MapView[K, V](null, null, mapSerializer.deserialize(source))
+ }
+
+ override def deserialize(reuse: MapView[K, V], source: DataInputView): MapView[K, V] =
+ deserialize(source)
+
+ override def copy(source: DataInputView, target: DataOutputView): Unit =
+ mapSerializer.copy(source, target)
+
+ override def canEqual(obj: Any): Boolean = obj != null && obj.getClass == getClass
+
+ override def hashCode(): Int = mapSerializer.hashCode()
+
+ override def equals(obj: Any): Boolean = canEqual(this) &&
+ mapSerializer.equals(obj.asInstanceOf[MapViewSerializer[_, _]].mapSerializer)
+
+ override def snapshotConfiguration(): TypeSerializerConfigSnapshot =
+ mapSerializer.snapshotConfiguration()
+
+ // copy and modified from MapSerializer.ensureCompatibility
+ override def ensureCompatibility(configSnapshot: TypeSerializerConfigSnapshot)
+ : CompatibilityResult[MapView[K, V]] = {
+
+ configSnapshot match {
+ case snapshot: MapSerializerConfigSnapshot[_, _] =>
+ val previousKvSerializersAndConfigs = snapshot.getNestedSerializersAndConfigs
+
+ val keyCompatResult = CompatibilityUtil.resolveCompatibilityResult(
+ previousKvSerializersAndConfigs.get(0).f0,
+ classOf[UnloadableDummyTypeSerializer[_]],
+ previousKvSerializersAndConfigs.get(0).f1,
+ mapSerializer.getKeySerializer)
+
+ val valueCompatResult = CompatibilityUtil.resolveCompatibilityResult(
+ previousKvSerializersAndConfigs.get(1).f0,
+ classOf[UnloadableDummyTypeSerializer[_]],
+ previousKvSerializersAndConfigs.get(1).f1,
+ mapSerializer.getValueSerializer)
+
+ if (!keyCompatResult.isRequiresMigration && !valueCompatResult.isRequiresMigration) {
+ CompatibilityResult.compatible[MapView[K, V]]
+ } else if (keyCompatResult.getConvertDeserializer != null
+ && valueCompatResult.getConvertDeserializer != null) {
+ CompatibilityResult.requiresMigration(
+ new MapViewSerializer[K, V](
+ new MapSerializer[K, V](
+ new TypeDeserializerAdapter[K](keyCompatResult.getConvertDeserializer),
+ new TypeDeserializerAdapter[V](valueCompatResult.getConvertDeserializer))
+ )
+ )
+ } else {
+ CompatibilityResult.requiresMigration[MapView[K, V]]
+ }
+
+ case _ => CompatibilityResult.requiresMigration[MapView[K, V]]
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfo.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfo.scala
new file mode 100644
index 0000000..ec5c222
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfo.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import org.apache.flink.api.common.ExecutionConfig
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.common.typeutils.base.MapSerializer
+import org.apache.flink.table.api.dataview.MapView
+
+/**
+ * [[MapView]] type information.
+ *
+ * @param keyType key type information
+ * @param valueType value type information
+ * @tparam K key type
+ * @tparam V value type
+ */
+class MapViewTypeInfo[K, V](
+ val keyType: TypeInformation[K],
+ val valueType: TypeInformation[V])
+ extends TypeInformation[MapView[K, V]] {
+
+ override def isBasicType = false
+
+ override def isTupleType = false
+
+ override def getArity = 1
+
+ override def getTotalFields = 1
+
+ override def getTypeClass: Class[MapView[K, V]] = classOf[MapView[K, V]]
+
+ override def isKeyType: Boolean = false
+
+ override def createSerializer(config: ExecutionConfig): TypeSerializer[MapView[K, V]] = {
+ val keySer = keyType.createSerializer(config)
+ val valueSer = valueType.createSerializer(config)
+ new MapViewSerializer[K, V](new MapSerializer[K, V](keySer, valueSer))
+ }
+
+ override def canEqual(obj: scala.Any): Boolean = obj != null && obj.getClass == getClass
+
+ override def hashCode(): Int = 31 * keyType.hashCode + valueType.hashCode
+
+ override def equals(obj: Any): Boolean = canEqual(obj) && {
+ obj match {
+ case other: MapViewTypeInfo[_, _] =>
+ keyType.equals(other.keyType) &&
+ valueType.equals(other.valueType)
+ case _ => false
+ }
+ }
+
+ override def toString: String = s"MapView<$keyType, $valueType>"
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfoFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfoFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfoFactory.scala
new file mode 100644
index 0000000..33c3ffe
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/MapViewTypeInfoFactory.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import java.lang.reflect.Type
+import java.util
+
+import org.apache.flink.api.common.typeinfo.{TypeInfoFactory, TypeInformation}
+import org.apache.flink.api.java.typeutils.GenericTypeInfo
+import org.apache.flink.table.api.dataview.MapView
+
+class MapViewTypeInfoFactory[K, V] extends TypeInfoFactory[MapView[K, V]] {
+
+ override def createTypeInfo(
+ t: Type,
+ genericParameters: util.Map[String, TypeInformation[_]]): TypeInformation[MapView[K, V]] = {
+
+ var keyType = genericParameters.get("K")
+ var valueType = genericParameters.get("V")
+
+ if (keyType == null) {
+ // we might can get the keyType later from the MapView constructor
+ keyType = new GenericTypeInfo(classOf[Any])
+ }
+
+ if (valueType == null) {
+ // we might can get the valueType later from the MapView constructor
+ valueType = new GenericTypeInfo(classOf[Any])
+ }
+
+ new MapViewTypeInfo[K, V](
+ keyType.asInstanceOf[TypeInformation[K]],
+ valueType.asInstanceOf[TypeInformation[V]])
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateListView.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateListView.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateListView.scala
new file mode 100644
index 0000000..70756ca
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateListView.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.dataview
+
+import java.util
+import java.lang.{Iterable => JIterable}
+
+import org.apache.flink.api.common.state._
+import org.apache.flink.table.api.dataview.ListView
+
+/**
+ * [[ListView]] use state backend.
+ *
+ * @param state list state
+ * @tparam T element type
+ */
+class StateListView[T](state: ListState[T]) extends ListView[T] {
+
+ override def get: JIterable[T] = state.get()
+
+ override def add(value: T): Unit = state.add(value)
+
+ override def addAll(list: util.List[T]): Unit = {
+ val iterator = list.iterator()
+ while (iterator.hasNext) {
+ state.add(iterator.next())
+ }
+ }
+
+ override def clear(): Unit = state.clear()
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateMapView.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateMapView.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateMapView.scala
new file mode 100644
index 0000000..22f5f0b
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/dataview/StateMapView.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.table.dataview
+
+import java.util
+import java.lang.{Iterable => JIterable}
+
+import org.apache.flink.api.common.state.MapState
+import org.apache.flink.table.api.dataview.MapView
+
+/**
+ * [[MapView]] use state backend.
+ *
+ * @param state map state
+ * @tparam K key type
+ * @tparam V value type
+ */
+class StateMapView[K, V](state: MapState[K, V]) extends MapView[K, V] {
+
+ override def get(key: K): V = state.get(key)
+
+ override def put(key: K, value: V): Unit = state.put(key, value)
+
+ override def putAll(map: util.Map[K, V]): Unit = state.putAll(map)
+
+ override def remove(key: K): Unit = state.remove(key)
+
+ override def contains(key: K): Boolean = state.contains(key)
+
+ override def entries: JIterable[util.Map.Entry[K, V]] = state.entries()
+
+ override def keys: JIterable[K] = state.keys()
+
+ override def values: JIterable[V] = state.values()
+
+ override def iterator: util.Iterator[util.Map.Entry[K, V]] = state.iterator()
+
+ override def clear(): Unit = state.clear()
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
index b44c28e..f53bcde 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
@@ -19,6 +19,7 @@
package org.apache.flink.table.functions.utils
+import java.util
import java.lang.{Integer => JInt, Long => JLong}
import java.lang.reflect.{Method, Modifier}
import java.sql.{Date, Time, Timestamp}
@@ -29,7 +30,10 @@ import org.apache.calcite.sql.`type`.SqlTypeName
import org.apache.calcite.sql.{SqlCallBinding, SqlFunction}
import org.apache.flink.api.common.functions.InvalidTypesException
import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.common.typeutils.CompositeType
+import org.apache.flink.api.java.typeutils.{PojoField, PojoTypeInfo, TypeExtractor}
+import org.apache.flink.table.api.dataview._
+import org.apache.flink.table.dataview._
import org.apache.flink.table.calcite.FlinkTypeFactory
import org.apache.flink.table.api.{TableEnvironment, TableException, ValidationException}
import org.apache.flink.table.expressions._
@@ -38,6 +42,8 @@ import org.apache.flink.table.functions.{AggregateFunction, ScalarFunction, Tabl
import org.apache.flink.table.plan.schema.FlinkTableFunctionImpl
import org.apache.flink.util.InstantiationUtil
+import scala.collection.mutable
+
object UserDefinedFunctionUtils {
/**
@@ -307,6 +313,111 @@ object UserDefinedFunctionUtils {
// ----------------------------------------------------------------------------------------------
/**
+ * Remove StateView fields from accumulator type information.
+ *
+ * @param index index of aggregate function
+ * @param aggFun aggregate function
+ * @param accType accumulator type information, only support pojo type
+ * @param isStateBackedDataViews is data views use state backend
+ * @return mapping of accumulator type information and data view config which contains id,
+ * field name and state descriptor
+ */
+ def removeStateViewFieldsFromAccTypeInfo(
+ index: Int,
+ aggFun: AggregateFunction[_, _],
+ accType: TypeInformation[_],
+ isStateBackedDataViews: Boolean)
+ : (TypeInformation[_], Option[Seq[DataViewSpec[_]]]) = {
+
+ /** Recursively checks if composite type includes a data view type. */
+ def includesDataView(ct: CompositeType[_]): Boolean = {
+ (0 until ct.getArity).exists(i =>
+ ct.getTypeAt(i) match {
+ case nestedCT: CompositeType[_] => includesDataView(nestedCT)
+ case t: TypeInformation[_] if t.getTypeClass == classOf[ListView[_]] => true
+ case t: TypeInformation[_] if t.getTypeClass == classOf[MapView[_, _]] => true
+ case _ => false
+ }
+ )
+ }
+
+ val acc = aggFun.createAccumulator()
+ accType match {
+ case pojoType: PojoTypeInfo[_] if pojoType.getArity > 0 =>
+ val arity = pojoType.getArity
+ val newPojoFields = new util.ArrayList[PojoField]()
+ val accumulatorSpecs = new mutable.ArrayBuffer[DataViewSpec[_]]
+ for (i <- 0 until arity) {
+ val pojoField = pojoType.getPojoFieldAt(i)
+ val field = pojoField.getField
+ val fieldName = field.getName
+ field.setAccessible(true)
+
+ pojoField.getTypeInformation match {
+ case ct: CompositeType[_] if includesDataView(ct) =>
+ throw new TableException(
+ "MapView and ListView only supported at first level of accumulators of Pojo type.")
+ case map: MapViewTypeInfo[_, _] =>
+ val mapView = field.get(acc).asInstanceOf[MapView[_, _]]
+ if (mapView != null) {
+ val keyTypeInfo = mapView.keyTypeInfo
+ val valueTypeInfo = mapView.valueTypeInfo
+ val newTypeInfo = if (keyTypeInfo != null && valueTypeInfo != null) {
+ new MapViewTypeInfo(keyTypeInfo, valueTypeInfo)
+ } else {
+ map
+ }
+
+ // create map view specs with unique id (used as state name)
+ var spec = MapViewSpec(
+ "agg" + index + "$" + fieldName,
+ field,
+ newTypeInfo)
+
+ accumulatorSpecs += spec
+ if (!isStateBackedDataViews) {
+ // add data view field if it is not backed by a state backend.
+ // data view fields which are backed by state backend are not serialized.
+ newPojoFields.add(new PojoField(field, newTypeInfo))
+ }
+ }
+
+ case list: ListViewTypeInfo[_] =>
+ val listView = field.get(acc).asInstanceOf[ListView[_]]
+ if (listView != null) {
+ val elementTypeInfo = listView.elementTypeInfo
+ val newTypeInfo = if (elementTypeInfo != null) {
+ new ListViewTypeInfo(elementTypeInfo)
+ } else {
+ list
+ }
+
+ // create list view specs with unique is (used as state name)
+ var spec = ListViewSpec(
+ "agg" + index + "$" + fieldName,
+ field,
+ newTypeInfo)
+
+ accumulatorSpecs += spec
+ if (!isStateBackedDataViews) {
+ // add data view field if it is not backed by a state backend.
+ // data view fields which are backed by state backend are not serialized.
+ newPojoFields.add(new PojoField(field, newTypeInfo))
+ }
+ }
+
+ case _ => newPojoFields.add(pojoField)
+ }
+ }
+ (new PojoTypeInfo(accType.getTypeClass, newPojoFields), Some(accumulatorSpecs))
+ case ct: CompositeType[_] if includesDataView(ct) =>
+ throw new TableException(
+ "MapView and ListView only supported in accumulators of POJO type.")
+ case _ => (accType, None)
+ }
+ }
+
+ /**
* Tries to infer the TypeInformation of an AggregateFunction's return type.
*
* @param aggregateFunction The AggregateFunction for which the return type is inferred.
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index 6304dc4..58940d0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -32,6 +32,7 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.functions.windowing.{AllWindowFunction, WindowFunction}
import org.apache.flink.streaming.api.windowing.windows.{Window => DataStreamWindow}
+import org.apache.flink.table.api.dataview.DataViewSpec
import org.apache.flink.table.api.{StreamQueryConfig, TableException}
import org.apache.flink.table.calcite.FlinkRelBuilder.NamedWindowProperty
import org.apache.flink.table.calcite.FlinkTypeFactory
@@ -82,11 +83,12 @@ object AggregateUtil {
isRowsClause: Boolean)
: ProcessFunction[CRow, CRow] = {
- val (aggFields, aggregates, accTypes) =
+ val (aggFields, aggregates, accTypes, accSpecs) =
transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputType,
- needRetraction = false)
+ needRetraction = false,
+ isStateBackedDataViews = true)
val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*)
@@ -107,7 +109,8 @@ object AggregateUtil {
outputArity,
needRetract = false,
needMerge = false,
- needReset = false
+ needReset = false,
+ accConfig = Some(accSpecs)
)
if (rowTimeIdx.isDefined) {
@@ -160,11 +163,12 @@ object AggregateUtil {
generateRetraction: Boolean,
consumeRetraction: Boolean): ProcessFunction[CRow, CRow] = {
- val (aggFields, aggregates, accTypes) =
+ val (aggFields, aggregates, accTypes, accSpecs) =
transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputRowType,
- consumeRetraction)
+ consumeRetraction,
+ isStateBackedDataViews = true)
val aggMapping = aggregates.indices.map(_ + groupings.length).toArray
@@ -185,7 +189,8 @@ object AggregateUtil {
outputArity,
consumeRetraction,
needMerge = false,
- needReset = false
+ needReset = false,
+ accConfig = Some(accSpecs)
)
new GroupAggProcessFunction(
@@ -223,11 +228,12 @@ object AggregateUtil {
: ProcessFunction[CRow, CRow] = {
val needRetract = true
- val (aggFields, aggregates, accTypes) =
+ val (aggFields, aggregates, accTypes, accSpecs) =
transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputType,
- needRetract)
+ needRetract,
+ isStateBackedDataViews = true)
val aggregationStateType: RowTypeInfo = new RowTypeInfo(accTypes: _*)
val inputRowType = CRowTypeInfo(inputTypeInfo)
@@ -249,7 +255,8 @@ object AggregateUtil {
outputArity,
needRetract,
needMerge = false,
- needReset = true
+ needReset = false,
+ accConfig = Some(accSpecs)
)
if (rowTimeIdx.isDefined) {
@@ -323,7 +330,7 @@ object AggregateUtil {
: MapFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, accTypes) = transformToAggregateFunctions(
+ val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputType,
needRetract)
@@ -380,7 +387,8 @@ object AggregateUtil {
outputArity,
needRetract,
needMerge = false,
- needReset = true
+ needReset = true,
+ None
)
new DataSetWindowAggMapFunction(
@@ -428,7 +436,7 @@ object AggregateUtil {
: RichGroupReduceFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, accTypes) = transformToAggregateFunctions(
+ val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract)
@@ -458,7 +466,8 @@ object AggregateUtil {
keysAndAggregatesArity + 1,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
new DataSetSlideTimeWindowAggReduceGroupFunction(
genFunction,
@@ -541,7 +550,7 @@ object AggregateUtil {
: RichGroupReduceFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, _) = transformToAggregateFunctions(
+ val (aggFieldIndexes, aggregates, _, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract)
@@ -561,7 +570,8 @@ object AggregateUtil {
outputType.getFieldCount,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
val genFinalAggFunction = generator.generateAggregations(
@@ -577,7 +587,8 @@ object AggregateUtil {
outputType.getFieldCount,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
val keysAndAggregatesArity = groupings.length + namedAggregates.length
@@ -686,7 +697,7 @@ object AggregateUtil {
groupings: Array[Int]): MapPartitionFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, accTypes) = transformToAggregateFunctions(
+ val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract)
@@ -718,7 +729,8 @@ object AggregateUtil {
groupings.length + aggregates.length + 2,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
new DataSetSessionWindowAggregatePreProcessor(
@@ -759,7 +771,7 @@ object AggregateUtil {
: GroupCombineFunction[Row, Row] = {
val needRetract = false
- val (aggFieldIndexes, aggregates, accTypes) = transformToAggregateFunctions(
+ val (aggFieldIndexes, aggregates, accTypes, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
physicalInputRowType,
needRetract)
@@ -792,7 +804,8 @@ object AggregateUtil {
groupings.length + aggregates.length + 2,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
new DataSetSessionWindowAggregatePreProcessor(
@@ -825,7 +838,7 @@ object AggregateUtil {
RichGroupReduceFunction[Row, Row]) = {
val needRetract = false
- val (aggInFields, aggregates, accTypes) = transformToAggregateFunctions(
+ val (aggInFields, aggregates, accTypes, _) = transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputType,
needRetract)
@@ -872,7 +885,8 @@ object AggregateUtil {
groupings.length + aggregates.length,
needRetract,
needMerge = false,
- needReset = true
+ needReset = true,
+ None
)
// compute mapping of forwarded grouping keys
@@ -898,7 +912,8 @@ object AggregateUtil {
outputType.getFieldCount,
needRetract,
needMerge = true,
- needReset = true
+ needReset = true,
+ None
)
(
@@ -921,7 +936,8 @@ object AggregateUtil {
outputType.getFieldCount,
needRetract,
needMerge = false,
- needReset = true
+ needReset = true,
+ None
)
(
@@ -996,7 +1012,7 @@ object AggregateUtil {
: (DataStreamAggFunction[CRow, Row, Row], RowTypeInfo, RowTypeInfo) = {
val needRetract = false
- val (aggFields, aggregates, accTypes) =
+ val (aggFields, aggregates, accTypes, _) =
transformToAggregateFunctions(
namedAggregates.map(_.getKey),
inputType,
@@ -1018,7 +1034,8 @@ object AggregateUtil {
outputArity,
needRetract,
needMerge,
- needReset = false
+ needReset = false,
+ None
)
val aggResultTypes = namedAggregates.map(a => FlinkTypeFactory.toTypeInfo(a.left.getType))
@@ -1159,8 +1176,12 @@ object AggregateUtil {
private def transformToAggregateFunctions(
aggregateCalls: Seq[AggregateCall],
inputType: RelDataType,
- needRetraction: Boolean)
- : (Array[Array[Int]], Array[TableAggregateFunction[_, _]], Array[TypeInformation[_]]) = {
+ needRetraction: Boolean,
+ isStateBackedDataViews: Boolean = false)
+ : (Array[Array[Int]],
+ Array[TableAggregateFunction[_, _]],
+ Array[TypeInformation[_]],
+ Array[Seq[DataViewSpec[_]]]) = {
// store the aggregate fields of each aggregate function, by the same order of aggregates.
val aggFieldIndexes = new Array[Array[Int]](aggregateCalls.size)
@@ -1398,14 +1419,28 @@ object AggregateUtil {
}
}
+ val accSpecs = new Array[Seq[DataViewSpec[_]]](aggregateCalls.size)
+
// create accumulator type information for every aggregate function
aggregates.zipWithIndex.foreach { case (agg, index) =>
- if (null == accTypes(index)) {
+ if (accTypes(index) != null) {
+ val (accType, specs) = removeStateViewFieldsFromAccTypeInfo(index,
+ agg,
+ accTypes(index),
+ isStateBackedDataViews)
+ if (specs.isDefined) {
+ accSpecs(index) = specs.get
+ accTypes(index) = accType
+ } else {
+ accSpecs(index) = Seq()
+ }
+ } else {
+ accSpecs(index) = Seq()
accTypes(index) = getAccumulatorTypeOfAggregateFunction(agg)
}
}
- (aggFieldIndexes, aggregates, accTypes)
+ (aggFieldIndexes, aggregates, accTypes, accSpecs)
}
private def createRowTypeForKeysAndAggregates(
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
index 5f48e09..7b20114 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.runtime.aggregate
-import org.apache.flink.api.common.functions.Function
+import org.apache.flink.api.common.functions.{Function, RuntimeContext}
import org.apache.flink.types.Row
/**
@@ -27,6 +27,14 @@ import org.apache.flink.types.Row
abstract class GeneratedAggregations extends Function {
/**
+ * Setup method for [[org.apache.flink.table.functions.AggregateFunction]].
+ * It can be used for initialization work. By default, this method does nothing.
+ *
+ * @param ctx The runtime context.
+ */
+ def open(ctx: RuntimeContext)
+
+ /**
* Sets the results of the aggregations (partial or final) to the output row.
* Final results are computed with the aggregation function.
* Partial results are the accumulators themselves.
@@ -100,6 +108,17 @@ abstract class GeneratedAggregations extends Function {
* aggregated results
*/
def resetAccumulator(accumulators: Row)
+
+ /**
+ * Cleanup for the accumulators.
+ */
+ def cleanup()
+
+ /**
+ * Tear-down method for [[org.apache.flink.table.functions.AggregateFunction]].
+ * It can be used for clean up work. By default, this method does nothing.
+ */
+ def close()
}
class SingleElementIterable[T] extends java.lang.Iterable[T] {
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 690a7c3..a476987 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -23,9 +23,8 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.apache.flink.api.common.state.ValueStateDescriptor
+import org.apache.flink.api.common.state.{StateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.api.common.state.ValueState
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.slf4j.{Logger, LoggerFactory}
@@ -65,6 +64,7 @@ class GroupAggProcessFunction(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
newRow = new CRow(function.createOutputRow(), true)
prevRow = new CRow(function.createOutputRow(), false)
@@ -162,7 +162,11 @@ class GroupAggProcessFunction(
if (needToCleanupState(timestamp)) {
cleanupState(state, cntState)
+ function.cleanup()
}
}
+ override def close(): Unit = {
+ function.close()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index ab3dc1d..5c28519 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -22,10 +22,7 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.api.common.state.ValueStateDescriptor
-import org.apache.flink.api.common.state.MapState
-import org.apache.flink.api.common.state.MapStateDescriptor
+import org.apache.flink.api.common.state._
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ListTypeInfo
import java.util.{ArrayList, List => JList}
@@ -71,6 +68,8 @@ class ProcTimeBoundedRangeOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
+
output = new CRow(function.createOutputRow(), true)
// We keep the elements received in a MapState indexed based on their ingestion time
@@ -121,6 +120,7 @@ class ProcTimeBoundedRangeOver(
if (needToCleanupState(timestamp)) {
// clean up and return
cleanupState(rowMapState, accumulatorState)
+ function.cleanup()
return
}
@@ -201,4 +201,7 @@ class ProcTimeBoundedRangeOver(
accumulatorState.update(accumulators)
}
+ override def close(): Unit = {
+ function.close()
+ }
}
[3/5] flink git commit: [FLINK-7206] [table] Add DataView to support
direct state access in AggregateFunction accumulators.
Posted by fh...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index ccc4b46..09d98ad 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -23,11 +23,8 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.types.Row
import org.apache.flink.util.{Collector, Preconditions}
-import org.apache.flink.api.common.state.ValueStateDescriptor
+import org.apache.flink.api.common.state._
import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.api.common.state.MapState
-import org.apache.flink.api.common.state.MapStateDescriptor
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.ListTypeInfo
import java.util.{List => JList}
@@ -75,6 +72,7 @@ class ProcTimeBoundedRowsOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
output = new CRow(function.createOutputRow(), true)
// We keep the elements received in a Map state keyed
@@ -194,6 +192,11 @@ class ProcTimeBoundedRowsOver(
if (needToCleanupState(timestamp)) {
cleanupState(rowMapState, accumulatorState, counterState, smallestTsState)
+ function.cleanup()
}
}
+
+ override def close(): Unit = {
+ function.close()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
index 7a7b44d..4fb5595 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
@@ -21,9 +21,8 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.types.Row
import org.apache.flink.util.Collector
-import org.apache.flink.api.common.state.ValueStateDescriptor
+import org.apache.flink.api.common.state.{StateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.api.common.state.ValueState
import org.apache.flink.table.api.StreamQueryConfig
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.runtime.types.CRow
@@ -56,6 +55,7 @@ class ProcTimeUnboundedOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
output = new CRow(function.createOutputRow(), true)
val stateDescriptor: ValueStateDescriptor[Row] =
@@ -97,6 +97,11 @@ class ProcTimeUnboundedOver(
if (needToCleanupState(timestamp)) {
cleanupState(state)
+ function.cleanup()
}
}
+
+ override def close(): Unit = {
+ function.close()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 8a0d682..1ee2693 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -78,6 +78,7 @@ class RowTimeBoundedRangeOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
output = new CRow(function.createOutputRow(), true)
@@ -158,6 +159,7 @@ class RowTimeBoundedRangeOver(
if (noRecordsToProcess) {
// we clean the state
cleanupState(dataState, accumulatorState, lastTriggeringTsState)
+ function.cleanup()
} else {
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
@@ -242,6 +244,10 @@ class RowTimeBoundedRangeOver(
// update cleanup timer
registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
+
+ override def close(): Unit = {
+ function.close()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index ba65846..60200bc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -84,6 +84,7 @@ class RowTimeBoundedRowsOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
output = new CRow(function.createOutputRow(), true)
@@ -168,6 +169,7 @@ class RowTimeBoundedRowsOver(
if (noRecordsToProcess) {
// We clean the state
cleanupState(dataState, accumulatorState, dataCountState, lastTriggeringTsState)
+ function.cleanup()
} else {
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
@@ -264,6 +266,10 @@ class RowTimeBoundedRowsOver(
// update cleanup timer
registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
+
+ override def close(): Unit = {
+ function.close()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 9210c00..c8183c9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -71,6 +71,7 @@ abstract class RowTimeUnboundedOver(
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance()
+ function.open(getRuntimeContext)
output = new CRow(function.createOutputRow(), true)
sortedTimestamps = new util.LinkedList[Long]()
@@ -150,6 +151,7 @@ abstract class RowTimeUnboundedOver(
if (noRecordsToProcess) {
// we clean the state
cleanupState(rowMapState, accumulatorState)
+ function.cleanup()
} else {
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
@@ -241,6 +243,9 @@ abstract class RowTimeUnboundedOver(
lastAccumulator: Row,
out: Collector[CRow]): Unit
+ override def close(): Unit = {
+ function.close()
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/java/org/apache/flink/table/runtime/utils/JavaUserDefinedAggFunctions.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/runtime/utils/JavaUserDefinedAggFunctions.java b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/runtime/utils/JavaUserDefinedAggFunctions.java
index 4d06bc2..14f812a 100644
--- a/flink-libraries/flink-table/src/test/java/org/apache/flink/table/runtime/utils/JavaUserDefinedAggFunctions.java
+++ b/flink-libraries/flink-table/src/test/java/org/apache/flink/table/runtime/utils/JavaUserDefinedAggFunctions.java
@@ -18,7 +18,10 @@
package org.apache.flink.table.runtime.utils;
+import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.api.dataview.ListView;
+import org.apache.flink.table.api.dataview.MapView;
import org.apache.flink.table.functions.AggregateFunction;
import java.util.Iterator;
@@ -135,4 +138,200 @@ public class JavaUserDefinedAggFunctions {
accumulator.count -= iWeight;
}
}
+
+ /**
+ * CountDistinct accumulator.
+ */
+ public static class CountDistinctAccum {
+ public MapView<String, Integer> map;
+ public long count;
+ }
+
+ /**
+ * CountDistinct aggregate.
+ */
+ public static class CountDistinct extends AggregateFunction<Long, CountDistinctAccum> {
+
+ @Override
+ public CountDistinctAccum createAccumulator() {
+ CountDistinctAccum accum = new CountDistinctAccum();
+ accum.map = new MapView<>(Types.STRING, Types.INT);
+ accum.count = 0L;
+ return accum;
+ }
+
+ //Overloaded accumulate method
+ public void accumulate(CountDistinctAccum accumulator, String id) {
+ try {
+ Integer cnt = accumulator.map.get(id);
+ if (cnt != null) {
+ cnt += 1;
+ accumulator.map.put(id, cnt);
+ } else {
+ accumulator.map.put(id, 1);
+ accumulator.count += 1;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ //Overloaded accumulate method
+ public void accumulate(CountDistinctAccum accumulator, long id) {
+ try {
+ Integer cnt = accumulator.map.get(String.valueOf(id));
+ if (cnt != null) {
+ cnt += 1;
+ accumulator.map.put(String.valueOf(id), cnt);
+ } else {
+ accumulator.map.put(String.valueOf(id), 1);
+ accumulator.count += 1;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Long getValue(CountDistinctAccum accumulator) {
+ return accumulator.count;
+ }
+ }
+
+ /**
+ * CountDistinct aggregate with merge.
+ */
+ public static class CountDistinctWithMerge extends CountDistinct {
+
+ //Overloaded merge method
+ public void merge(CountDistinctAccum acc, Iterable<CountDistinctAccum> it) {
+ Iterator<CountDistinctAccum> iter = it.iterator();
+ while (iter.hasNext()) {
+ CountDistinctAccum mergeAcc = iter.next();
+ acc.count += mergeAcc.count;
+
+ try {
+ Iterator<String> itrMap = mergeAcc.map.keys().iterator();
+ while (itrMap.hasNext()) {
+ String key = itrMap.next();
+ Integer cnt = mergeAcc.map.get(key);
+ if (acc.map.contains(key)) {
+ acc.map.put(key, acc.map.get(key) + cnt);
+ } else {
+ acc.map.put(key, cnt);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * CountDistinct aggregate with merge and reset.
+ */
+ public static class CountDistinctWithMergeAndReset extends CountDistinctWithMerge {
+
+ //Overloaded retract method
+ public void resetAccumulator(CountDistinctAccum acc) {
+ acc.map.clear();
+ acc.count = 0;
+ }
+ }
+
+ /**
+ * CountDistinct aggregate with retract.
+ */
+ public static class CountDistinctWithRetractAndReset extends CountDistinct {
+
+ //Overloaded retract method
+ public void retract(CountDistinctAccum accumulator, long id) {
+ try {
+ Integer cnt = accumulator.map.get(String.valueOf(id));
+ if (cnt != null) {
+ cnt -= 1;
+ if (cnt <= 0) {
+ accumulator.map.remove(String.valueOf(id));
+ accumulator.count -= 1;
+ } else {
+ accumulator.map.put(String.valueOf(id), cnt);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ //Overloaded retract method
+ public void resetAccumulator(CountDistinctAccum acc) {
+ acc.map.clear();
+ acc.count = 0;
+ }
+ }
+
+ /**
+ * Accumulator for test DataView.
+ */
+ public static class DataViewTestAccum {
+ public MapView<String, Integer> map;
+ public MapView<String, Integer> map2; // for test not initialized
+ public long count;
+ private ListView<Long> list = new ListView<>(Types.LONG);
+
+ public ListView<Long> getList() {
+ return list;
+ }
+
+ public void setList(ListView<Long> list) {
+ this.list = list;
+ }
+ }
+
+ public static boolean isCloseCalled = false;
+
+ /**
+ * Aggregate for test DataView.
+ */
+ public static class DataViewTestAgg extends AggregateFunction<Long, DataViewTestAccum> {
+ @Override
+ public DataViewTestAccum createAccumulator() {
+ DataViewTestAccum accum = new DataViewTestAccum();
+ accum.map = new MapView<>(Types.STRING, Types.INT);
+ accum.count = 0L;
+ return accum;
+ }
+
+ // Overloaded accumulate method
+ public void accumulate(DataViewTestAccum accumulator, String a, long b) {
+ try {
+ if (!accumulator.map.contains(a)) {
+ accumulator.map.put(a, 1);
+ accumulator.count++;
+ }
+
+ accumulator.list.add(b);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Long getValue(DataViewTestAccum accumulator) {
+ long sum = accumulator.count;
+ try {
+ for (Long value : accumulator.list.get()) {
+ sum += value;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return sum;
+ }
+
+ @Override
+ public void close() {
+ isCloseCalled = true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/ListViewSerializerTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/ListViewSerializerTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/ListViewSerializerTest.scala
new file mode 100644
index 0000000..3f70bce
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/ListViewSerializerTest.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import java.lang.Long
+import java.util.Random
+
+import org.apache.flink.api.common.typeutils.base.{ListSerializer, LongSerializer}
+import org.apache.flink.api.common.typeutils.{SerializerTestBase, TypeSerializer}
+import org.apache.flink.table.api.dataview.ListView
+
+/**
+ * A test for the [[ListViewSerializer]].
+ */
+class ListViewSerializerTest extends SerializerTestBase[ListView[Long]] {
+
+ override protected def createSerializer(): TypeSerializer[ListView[Long]] = {
+ val listSerializer = new ListSerializer[Long](LongSerializer.INSTANCE)
+ new ListViewSerializer[Long](listSerializer)
+ }
+
+ override protected def getLength: Int = -1
+
+ override protected def getTypeClass: Class[ListView[Long]] = classOf[ListView[Long]]
+
+ override protected def getTestData: Array[ListView[Long]] = {
+ val rnd = new Random(321332)
+
+ // empty
+ val listview1 = new ListView[Long]()
+
+ // single element
+ val listview2 = new ListView[Long]()
+ listview2.add(12345L)
+
+ // multiple elements
+ val listview3 = new ListView[Long]()
+ var i = 0
+ while (i < rnd.nextInt(200)) {
+ listview3.add(rnd.nextLong)
+ i += 1
+ }
+
+ Array[ListView[Long]](listview1, listview2, listview3)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/MapViewSerializerTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/MapViewSerializerTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/MapViewSerializerTest.scala
new file mode 100644
index 0000000..15f9b02
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/dataview/MapViewSerializerTest.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.dataview
+
+import java.lang.Long
+import java.util.Random
+
+import org.apache.flink.api.common.typeutils.base.{LongSerializer, MapSerializer, StringSerializer}
+import org.apache.flink.api.common.typeutils.{SerializerTestBase, TypeSerializer}
+import org.apache.flink.table.api.dataview.MapView
+
+/**
+ * A test for the [[MapViewSerializer]].
+ */
+class MapViewSerializerTest extends SerializerTestBase[MapView[Long, String]] {
+
+ override protected def createSerializer(): TypeSerializer[MapView[Long, String]] = {
+ val mapSerializer = new MapSerializer[Long, String](LongSerializer.INSTANCE,
+ StringSerializer.INSTANCE)
+ new MapViewSerializer[Long, String](mapSerializer)
+ }
+
+ override protected def getLength: Int = -1
+
+ override protected def getTypeClass: Class[MapView[Long, String]] =
+ classOf[MapView[Long, String]]
+
+ override protected def getTestData: Array[MapView[Long, String]] = {
+ val rnd = new Random(321654)
+
+ // empty
+ val mapview1 = new MapView[Long, String]()
+
+ // single element
+ val mapview2 = new MapView[Long, String]()
+ mapview2.put(12345L, "12345L")
+
+ // multiple elements
+ val mapview3 = new MapView[Long, String]()
+ var i = 0
+ while (i < rnd.nextInt(200)) {
+ mapview3.put(rnd.nextLong, Long.toString(rnd.nextLong))
+ i += 1
+ }
+
+ // null-value maps
+ val mapview4 = new MapView[Long, String]()
+ mapview4.put(999L, null)
+
+ Array[MapView[Long, String]](mapview1, mapview2, mapview3, mapview4)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala
index d563f96..cf96d19 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/AggregateITCase.scala
@@ -23,7 +23,7 @@ import java.math.BigDecimal
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.util.CollectionDataSets
import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvgWithMergeAndReset
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinctWithMergeAndReset, WeightedAvgWithMergeAndReset}
import org.apache.flink.table.api.scala._
import org.apache.flink.table.functions.aggfunctions.CountAggFunction
import org.apache.flink.table.runtime.utils.TableProgramsCollectionTestBase
@@ -226,13 +226,14 @@ class AggregationsITCase(
val tEnv = TableEnvironment.getTableEnvironment(env, config)
val countFun = new CountAggFunction
val wAvgFun = new WeightedAvgWithMergeAndReset
+ val countDistinct = new CountDistinctWithMergeAndReset
val t = CollectionDataSets.get3TupleDataSet(env).toTable(tEnv, 'a, 'b, 'c)
.groupBy('b)
- .select('b, 'a.sum, countFun('c), wAvgFun('b, 'a), wAvgFun('a, 'a))
+ .select('b, 'a.sum, countFun('c), wAvgFun('b, 'a), wAvgFun('a, 'a), countDistinct('c))
- val expected = "1,1,1,1,1\n" + "2,5,2,2,2\n" + "3,15,3,3,5\n" + "4,34,4,4,8\n" +
- "5,65,5,5,13\n" + "6,111,6,6,18\n"
+ val expected = "1,1,1,1,1,1\n" + "2,5,2,2,2,2\n" + "3,15,3,3,5,3\n" + "4,34,4,4,8,4\n" +
+ "5,65,5,5,13,5\n" + "6,111,6,6,18,6\n"
val results = t.toDataSet[Row].collect()
TestBaseUtils.compareResultAsText(results.asJava, expected)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
index 04aada6..67164b7 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/HarnessTestBase.scala
@@ -170,6 +170,14 @@ class HarnessTestBase {
| return new org.apache.flink.types.Row(5);
| }
|
+ | public void open(org.apache.flink.api.common.functions.RuntimeContext ctx) {
+ | }
+ |
+ | public void cleanup() {
+ | }
+ |
+ | public void close() {
+ | }
|/******* This test does not use the following methods *******/
| public org.apache.flink.types.Row mergeAccumulatorsPair(
| org.apache.flink.types.Row a,
@@ -282,6 +290,15 @@ class HarnessTestBase {
| public final void resetAccumulator(
| org.apache.flink.types.Row accs) {
| }
+ |
+ | public void open(org.apache.flink.api.common.functions.RuntimeContext ctx) {
+ | }
+ |
+ | public void cleanup() {
+ | }
+ |
+ | public void close() {
+ | }
|}
|""".stripMargin
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala
index 744ac46..eb3d37f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/AggregateITCase.scala
@@ -24,7 +24,8 @@ import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.table.api.scala._
import org.apache.flink.table.runtime.utils.StreamITCase.RetractingSink
import org.apache.flink.table.api.{StreamQueryConfig, TableEnvironment}
-import org.apache.flink.table.runtime.utils.{StreamITCase, StreamTestData, StreamingWithStateTestBase}
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, DataViewTestAgg}
+import org.apache.flink.table.runtime.utils.{JavaUserDefinedAggFunctions, StreamITCase, StreamTestData, StreamingWithStateTestBase}
import org.apache.flink.types.Row
import org.junit.Assert.assertEquals
import org.junit.Test
@@ -154,4 +155,42 @@ class AggregateITCase extends StreamingWithStateTestBase {
"12,3,5,1", "5,3,4,2")
assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
}
+
+ @Test
+ def testGroupAggregateWithStateBackend(): Unit = {
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.setStateBackend(getStateBackend)
+ val tEnv = TableEnvironment.getTableEnvironment(env)
+ StreamITCase.clear
+
+ val data = new mutable.MutableList[(Int, Long, String)]
+ data.+=((1, 1L, "A"))
+ data.+=((2, 2L, "B"))
+ data.+=((3, 2L, "B"))
+ data.+=((4, 3L, "C"))
+ data.+=((5, 3L, "C"))
+ data.+=((6, 3L, "C"))
+ data.+=((7, 4L, "B"))
+ data.+=((8, 4L, "A"))
+ data.+=((9, 4L, "D"))
+ data.+=((10, 4L, "E"))
+ data.+=((11, 5L, "A"))
+ data.+=((12, 5L, "B"))
+
+ val distinct = new CountDistinct
+ val testAgg = new DataViewTestAgg
+ val t = env.fromCollection(data).toTable(tEnv, 'a, 'b, 'c)
+ .groupBy('b)
+ .select('b, distinct('c), testAgg('c, 'b))
+
+ val results = t.toRetractStream[Row](queryConfig)
+ results.addSink(new StreamITCase.RetractingSink)
+ env.execute()
+
+ val expected = List("1,1,2", "2,1,5", "3,1,10", "4,4,20", "5,2,12")
+ assertEquals(expected.sorted, StreamITCase.retractedResults.sorted)
+
+ // verify agg close is called
+ assert(JavaUserDefinedAggFunctions.isCloseCalled)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
index 1561da0..f6e739e 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/GroupWindowITCase.scala
@@ -29,7 +29,7 @@ import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.util.StreamingMultipleProgramsTestBase
-import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithMerge, WeightedAvg, WeightedAvgWithMerge}
import org.apache.flink.table.functions.aggfunctions.CountAggFunction
import org.apache.flink.table.runtime.stream.table.GroupWindowITCase._
import org.apache.flink.table.runtime.utils.StreamITCase
@@ -75,19 +75,21 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvg
+ val countDistinct = new CountDistinct
val windowedTable = table
.window(Slide over 2.rows every 1.rows on 'proctime as 'w)
.groupBy('w, 'string)
.select('string, countFun('int), 'int.avg,
- weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int),
+ countDistinct('long))
val results = windowedTable.toAppendStream[Row](queryConfig)
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
- val expected = Seq("Hello world,1,3,8,3", "Hello world,2,3,12,3", "Hello,1,2,2,2",
- "Hello,2,2,3,2", "Hi,1,1,1,1")
+ val expected = Seq("Hello world,1,3,8,3,1", "Hello world,2,3,12,3,2", "Hello,1,2,2,2,1",
+ "Hello,2,2,3,2,2", "Hi,1,1,1,1,1")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -112,6 +114,7 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvgWithMerge
+ val countDistinct = new CountDistinctWithMerge
val stream = env
.fromCollection(sessionWindowTestdata)
@@ -122,13 +125,14 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
.window(Session withGap 5.milli on 'rowtime as 'w)
.groupBy('w, 'string)
.select('string, countFun('int), 'int.avg,
- weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int),
+ countDistinct('long))
val results = windowedTable.toAppendStream[Row]
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
- val expected = Seq("Hello World,1,9,9,9", "Hello,1,16,16,16", "Hello,4,3,5,5")
+ val expected = Seq("Hello World,1,9,9,9,1", "Hello,1,16,16,16,1", "Hello,4,3,5,5,4")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -143,18 +147,21 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
val table = stream.toTable(tEnv, 'long, 'int, 'string, 'proctime.proctime)
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvg
+ val countDistinct = new CountDistinct
val windowedTable = table
.window(Tumble over 2.rows on 'proctime as 'w)
.groupBy('w)
.select(countFun('string), 'int.avg,
- weightAvgFun('long, 'int), weightAvgFun('int, 'int))
+ weightAvgFun('long, 'int), weightAvgFun('int, 'int),
+ countDistinct('long)
+ )
val results = windowedTable.toAppendStream[Row](queryConfig)
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
- val expected = Seq("2,1,1,1", "2,2,6,2")
+ val expected = Seq("2,1,1,1,2", "2,2,6,2,2")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -171,22 +178,24 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
val table = stream.toTable(tEnv, 'long, 'int, 'string, 'rowtime.rowtime)
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvg
+ val countDistinct = new CountDistinct
val windowedTable = table
.window(Tumble over 5.milli on 'rowtime as 'w)
.groupBy('w, 'string)
.select('string, countFun('string), 'int.avg, weightAvgFun('long, 'int),
- weightAvgFun('int, 'int), 'int.min, 'int.max, 'int.sum, 'w.start, 'w.end)
+ weightAvgFun('int, 'int), 'int.min, 'int.max, 'int.sum, 'w.start, 'w.end,
+ countDistinct('long))
val results = windowedTable.toAppendStream[Row]
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = Seq(
- "Hello world,1,3,8,3,3,3,3,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01",
- "Hello world,1,3,16,3,3,3,3,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02",
- "Hello,2,2,3,2,2,2,4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005",
- "Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005")
+ "Hello world,1,3,8,3,3,3,3,1970-01-01 00:00:00.005,1970-01-01 00:00:00.01,1",
+ "Hello world,1,3,16,3,3,3,3,1970-01-01 00:00:00.015,1970-01-01 00:00:00.02,1",
+ "Hello,2,2,3,2,2,2,4,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,2",
+ "Hi,1,1,1,1,1,1,1,1970-01-01 00:00:00.0,1970-01-01 00:00:00.005,1")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -208,17 +217,18 @@ class GroupWindowITCase extends StreamingMultipleProgramsTestBase {
val table = stream.toTable(tEnv, 'long, 'int, 'string, 'int2, 'int3, 'proctime.proctime)
val weightAvgFun = new WeightedAvg
+ val countDistinct = new CountDistinct
val windowedTable = table
.window(Slide over 2.rows every 1.rows on 'proctime as 'w)
.groupBy('w, 'int2, 'int3, 'string)
- .select(weightAvgFun('long, 'int))
+ .select(weightAvgFun('long, 'int), countDistinct('long))
val results = windowedTable.toAppendStream[Row]
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
- val expected = Seq("12", "8", "2", "3", "1")
+ val expected = Seq("12,2", "8,1", "2,1", "3,2", "1,1")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1fc0b641/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
index 73484d2..54971b2 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/OverWindowITCase.scala
@@ -25,7 +25,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.table.api.TableEnvironment
-import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.WeightedAvg
+import org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions.{CountDistinct, CountDistinctWithRetractAndReset, WeightedAvg}
import org.apache.flink.table.runtime.utils.JavaUserDefinedScalarFunctions.JavaFunc0
import org.apache.flink.table.api.scala._
import org.apache.flink.table.functions.aggfunctions.CountAggFunction
@@ -51,6 +51,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
(6L, 6, "Hello"),
(7L, 7, "Hello World"),
(8L, 8, "Hello World"),
+ (8L, 8, "Hello World"),
(20L, 20, "Hello World"))
val env = StreamExecutionEnvironment.getExecutionEnvironment
@@ -62,20 +63,24 @@ class OverWindowITCase extends StreamingWithStateTestBase {
val table = stream.toTable(tEnv, 'a, 'b, 'c, 'proctime.proctime)
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvg
+ val countDist = new CountDistinct
val windowedTable = table
.window(
Over partitionBy 'c orderBy 'proctime preceding UNBOUNDED_ROW as 'w)
- .select('c, countFun('b) over 'w as 'mycount, weightAvgFun('a, 'b) over 'w as 'wAvg)
- .select('c, 'mycount, 'wAvg)
+ .select('c,
+ countFun('b) over 'w as 'mycount,
+ weightAvgFun('a, 'b) over 'w as 'wAvg,
+ countDist('a) over 'w as 'countDist)
+ .select('c, 'mycount, 'wAvg, 'countDist)
val results = windowedTable.toAppendStream[Row]
results.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = Seq(
- "Hello World,1,7", "Hello World,2,7", "Hello World,3,14",
- "Hello,1,1", "Hello,2,1", "Hello,3,2", "Hello,4,3", "Hello,5,3", "Hello,6,4")
+ "Hello World,1,7,1", "Hello World,2,7,2", "Hello World,3,7,2", "Hello World,4,13,3",
+ "Hello,1,1,1", "Hello,2,1,2", "Hello,3,2,3", "Hello,4,3,4", "Hello,5,3,5", "Hello,6,4,6")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -112,6 +117,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
val countFun = new CountAggFunction
val weightAvgFun = new WeightedAvg
val plusOne = new JavaFunc0
+ val countDist = new CountDistinct
val windowedTable = table
.window(Over partitionBy 'a orderBy 'rowtime preceding UNBOUNDED_RANGE following
@@ -128,26 +134,27 @@ class OverWindowITCase extends StreamingWithStateTestBase {
'b.max over 'w,
'b.min over 'w,
('b.min over 'w).abs(),
- weightAvgFun('b, 'a) over 'w)
+ weightAvgFun('b, 'a) over 'w,
+ countDist('c) over 'w as 'countDist)
val result = windowedTable.toAppendStream[Row]
result.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = mutable.MutableList(
- "1,1,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
- "1,2,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
- "1,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
- "1,1,Hi,7,SUM:7,4,5,5,[1, 3],1,3,1,1,1",
- "2,1,Hello,1,SUM:1,1,2,2,[1, 1],1,1,1,1,1",
- "2,2,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
- "2,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2",
- "1,4,Hello world,11,SUM:11,5,6,6,[2, 4],2,4,1,1,2",
- "1,5,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
- "1,6,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
- "1,7,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3",
- "2,4,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3",
- "2,5,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3"
+ "1,1,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2,2",
+ "1,2,Hello,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2,2",
+ "1,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2,2",
+ "1,1,Hi,7,SUM:7,4,5,5,[1, 3],1,3,1,1,1,3",
+ "2,1,Hello,1,SUM:1,1,2,2,[1, 1],1,1,1,1,1,1",
+ "2,2,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2,2",
+ "2,3,Hello world,6,SUM:6,3,4,4,[2, 3],2,3,1,1,2,2",
+ "1,4,Hello world,11,SUM:11,5,6,6,[2, 4],2,4,1,1,2,3",
+ "1,5,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3,3",
+ "1,6,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3,3",
+ "1,7,Hello world,29,SUM:29,8,9,9,[3, 7],3,7,1,1,3,3",
+ "2,4,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3,2",
+ "2,5,Hello world,15,SUM:15,5,6,6,[3, 5],3,5,1,1,3,2"
)
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
@@ -179,32 +186,33 @@ class OverWindowITCase extends StreamingWithStateTestBase {
env.setParallelism(1)
StreamITCase.testResults = mutable.MutableList()
+ val countDist = new CountDistinctWithRetractAndReset
val stream = env.fromCollection(data)
val table = stream.toTable(tEnv, 'a, 'b, 'c, 'd, 'e, 'proctime.proctime)
val windowedTable = table
.window(Over partitionBy 'a orderBy 'proctime preceding 4.rows following CURRENT_ROW as 'w)
- .select('a, 'c.sum over 'w, 'c.min over 'w)
+ .select('a, 'c.sum over 'w, 'c.min over 'w, countDist('e) over 'w)
val result = windowedTable.toAppendStream[Row]
result.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = mutable.MutableList(
- "1,0,0",
- "2,1,1",
- "2,3,1",
- "3,3,3",
- "3,7,3",
- "3,12,3",
- "4,6,6",
- "4,13,6",
- "4,21,6",
- "4,30,6",
- "5,10,10",
- "5,21,10",
- "5,33,10",
- "5,46,10",
- "5,60,10")
+ "1,0,0,1",
+ "2,1,1,1",
+ "2,3,1,2",
+ "3,3,3,1",
+ "3,7,3,1",
+ "3,12,3,2",
+ "4,6,6,1",
+ "4,13,6,2",
+ "4,21,6,2",
+ "4,30,6,2",
+ "5,10,10,1",
+ "5,21,10,2",
+ "5,33,10,2",
+ "5,46,10,3",
+ "5,60,10,3")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -240,25 +248,27 @@ class OverWindowITCase extends StreamingWithStateTestBase {
val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.clear
+ val countDist = new CountDistinctWithRetractAndReset
val table = env.addSource[(Long, Int, String)](
new RowTimeSourceFunction[(Long, Int, String)](data))
.toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
val windowedTable = table
.window(Over partitionBy 'c orderBy 'rowtime preceding 2.rows following CURRENT_ROW as 'w)
- .select('c, 'a, 'a.count over 'w, 'a.sum over 'w)
+ .select('c, 'a, 'a.count over 'w, 'a.sum over 'w, countDist('a) over 'w)
val result = windowedTable.toAppendStream[Row]
result.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = mutable.MutableList(
- "Hello,1,1,1", "Hello,1,2,2", "Hello,1,3,3",
- "Hello,2,3,4", "Hello,2,3,5", "Hello,2,3,6",
- "Hello,3,3,7", "Hello,4,3,9", "Hello,5,3,12",
- "Hello,6,3,15",
- "Hello World,7,1,7", "Hello World,7,2,14", "Hello World,7,3,21",
- "Hello World,7,3,21", "Hello World,8,3,22", "Hello World,20,3,35")
+ "Hello,1,1,1,1", "Hello,1,2,2,1", "Hello,1,3,3,1",
+ "Hello,2,3,4,2", "Hello,2,3,5,2", "Hello,2,3,6,1",
+ "Hello,3,3,7,2", "Hello,4,3,9,3", "Hello,5,3,12,3",
+ "Hello,6,3,15,3",
+ "Hello World,7,1,7,1", "Hello World,7,2,14,1", "Hello World,7,3,21,1",
+ "Hello World,7,3,21,1", "Hello World,8,3,22,2", "Hello World,20,3,35,3")
+
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
@@ -302,6 +312,7 @@ class OverWindowITCase extends StreamingWithStateTestBase {
val tEnv = TableEnvironment.getTableEnvironment(env)
StreamITCase.clear
+ val countDist = new CountDistinctWithRetractAndReset
val table = env.addSource[(Long, Int, String)](
new RowTimeSourceFunction[(Long, Int, String)](data))
.toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
@@ -309,23 +320,24 @@ class OverWindowITCase extends StreamingWithStateTestBase {
val windowedTable = table
.window(
Over partitionBy 'c orderBy 'rowtime preceding 1.seconds following CURRENT_RANGE as 'w)
- .select('c, 'b, 'a.count over 'w, 'a.sum over 'w)
+ .select('c, 'b, 'a.count over 'w, 'a.sum over 'w, countDist('a) over 'w)
val result = windowedTable.toAppendStream[Row]
result.addSink(new StreamITCase.StringSink[Row])
env.execute()
val expected = mutable.MutableList(
- "Hello,1,1,1", "Hello,15,2,2", "Hello,16,3,3",
- "Hello,2,6,9", "Hello,3,6,9", "Hello,2,6,9",
- "Hello,3,4,9",
- "Hello,4,2,7",
- "Hello,5,2,9",
- "Hello,6,2,11", "Hello,65,2,12",
- "Hello,9,2,12", "Hello,9,2,12", "Hello,18,3,18",
- "Hello World,7,1,7", "Hello World,17,3,21", "Hello World,77,3,21", "Hello World,18,1,7",
- "Hello World,8,2,15",
- "Hello World,20,1,20")
+ "Hello,1,1,1,1", "Hello,15,2,2,1", "Hello,16,3,3,1",
+ "Hello,2,6,9,2", "Hello,3,6,9,2", "Hello,2,6,9,2",
+ "Hello,3,4,9,2",
+ "Hello,4,2,7,2",
+ "Hello,5,2,9,2",
+ "Hello,6,2,11,2", "Hello,65,2,12,1",
+ "Hello,9,2,12,1", "Hello,9,2,12,1", "Hello,18,3,18,1",
+ "Hello World,7,1,7,1", "Hello World,17,3,21,1",
+ "Hello World,77,3,21,1", "Hello World,18,1,7,1",
+ "Hello World,8,2,15,2",
+ "Hello World,20,1,20,1")
assertEquals(expected.sorted, StreamITCase.testResults.sorted)
}
}