You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2018/11/19 13:44:14 UTC
[flink] branch master updated: [FLINK-10670] [table] Fix Correlate
codegen error
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new ea9507d [FLINK-10670] [table] Fix Correlate codegen error
ea9507d is described below
commit ea9507dbc2479d77fd85f431c38589263cb07fa0
Author: Xpray <le...@gmail.com>
AuthorDate: Thu Oct 25 00:02:13 2018 +0800
[FLINK-10670] [table] Fix Correlate codegen error
This closes #6923.
---
.../apache/flink/table/codegen/CodeGenerator.scala | 14 +++++++-------
.../table/codegen/CollectorCodeGenerator.scala | 11 ++++++++---
.../flink/table/plan/nodes/CommonCorrelate.scala | 15 +++------------
.../runtime/stream/table/CorrelateITCase.scala | 21 +++++++++++++++++++++
4 files changed, 39 insertions(+), 22 deletions(-)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index b0d3737..fdb0d50 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -108,39 +108,39 @@ abstract class CodeGenerator(
// set of member statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- protected val reusableMemberStatements: mutable.LinkedHashSet[String] =
+ val reusableMemberStatements: mutable.LinkedHashSet[String] =
mutable.LinkedHashSet[String]()
// set of constructor statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- protected val reusableInitStatements: mutable.LinkedHashSet[String] =
+ val reusableInitStatements: mutable.LinkedHashSet[String] =
mutable.LinkedHashSet[String]()
// set of open statements for RichFunction that will be added only once
// we use a LinkedHashSet to keep the insertion order
- protected val reusableOpenStatements: mutable.LinkedHashSet[String] =
+ val reusableOpenStatements: mutable.LinkedHashSet[String] =
mutable.LinkedHashSet[String]()
// set of close statements for RichFunction that will be added only once
// we use a LinkedHashSet to keep the insertion order
- protected val reusableCloseStatements: mutable.LinkedHashSet[String] =
+ val reusableCloseStatements: mutable.LinkedHashSet[String] =
mutable.LinkedHashSet[String]()
// set of statements that will be added only once per record;
// code should only update member variables because local variables are not accessible if
// the code needs to be split;
// we use a LinkedHashSet to keep the insertion order
- protected val reusablePerRecordStatements: mutable.LinkedHashSet[String] =
+ val reusablePerRecordStatements: mutable.LinkedHashSet[String] =
mutable.LinkedHashSet[String]()
// map of initial input unboxing expressions that will be added only once
// (inputTerm, index) -> expr
- protected val reusableInputUnboxingExprs: mutable.Map[(String, Int), GeneratedExpression] =
+ val reusableInputUnboxingExprs: mutable.Map[(String, Int), GeneratedExpression] =
mutable.Map[(String, Int), GeneratedExpression]()
// set of constructor statements that will be added only once
// we use a LinkedHashSet to keep the insertion order
- protected val reusableConstructorStatements: mutable.LinkedHashSet[(String, String)] =
+ val reusableConstructorStatements: mutable.LinkedHashSet[(String, String)] =
mutable.LinkedHashSet[(String, String)]()
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala
index 85d858f..ddc13c5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CollectorCodeGenerator.scala
@@ -59,13 +59,14 @@ class CollectorCodeGenerator(
* valid Java class identifier.
* @param bodyCode body code for the collector method
* @param collectedType The type information of the element collected by the collector
+ * @param filterGenerator generator containing context information for the generated body code
* @return instance of GeneratedCollector
*/
def generateTableFunctionCollector(
name: String,
bodyCode: String,
collectedType: TypeInformation[Any],
- codeGenerator: CodeGenerator)
+ filterGenerator: CodeGenerator)
: GeneratedCollector = {
val className = newName(name)
@@ -86,6 +87,10 @@ class CollectorCodeGenerator(
s"$input2TypeClass $input2Term" // local variable
}
+ reusableMemberStatements ++= filterGenerator.reusableMemberStatements
+ reusableInitStatements ++= filterGenerator.reusableInitStatements
+ reusablePerRecordStatements ++= filterGenerator.reusablePerRecordStatements
+
val funcCode = j"""
|public class $className extends ${classOf[TableFunctionCollector[_]].getCanonicalName} {
|
@@ -98,7 +103,7 @@ class CollectorCodeGenerator(
|
| @Override
| public void open(${classOf[Configuration].getCanonicalName} parameters) throws Exception {
- | ${codeGenerator.reuseOpenCode()}
+ | ${filterGenerator.reuseOpenCode()}
| }
|
| @Override
@@ -113,7 +118,7 @@ class CollectorCodeGenerator(
|
| @Override
| public void close() throws Exception {
- | ${codeGenerator.reuseCloseCode()}
+ | ${filterGenerator.reuseCloseCode()}
| }
|}
|""".stripMargin
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 53adf6b..b9b5a3e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.plan.nodes
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexShuttle}
+import org.apache.calcite.rex.{RexCall, RexNode}
import org.apache.calcite.sql.SemiJoinType
import org.apache.flink.api.common.functions.Function
import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -149,18 +149,9 @@ trait CommonCorrelate {
|getCollector().collect(${crossResultExpr.resultTerm});
|""".stripMargin
} else {
-
- // adjust indices of InputRefs to adhere to schema expected by generator
- val changeInputRefIndexShuttle = new RexShuttle {
- override def visitInputRef(inputRef: RexInputRef): RexNode = {
- new RexInputRef(inputSchema.arity + inputRef.getIndex, inputRef.getType)
- }
- }
- // Run generateExpression to add init statements (ScalarFunctions) of condition to generator.
- // The generated expression is discarded.
- generator.generateExpression(condition.get.accept(changeInputRefIndexShuttle))
-
filterGenerator.input1Term = filterGenerator.input2Term
+ // generating filter expressions might add init statements and member fields
+ // that need to be combined with the statements of the collector code generator
val filterCondition = filterGenerator.generateExpression(condition.get)
s"""
|${filterGenerator.reuseInputUnboxingCode()}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala
index f4b1f80..60e03d4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/table/CorrelateITCase.scala
@@ -284,6 +284,27 @@ class CorrelateITCase extends AbstractTestBase {
)
}
+ @Test
+ def testTableFunctionCollectorInit(): Unit = {
+ val t = testData(env).toTable(tEnv).as('a, 'b, 'c)
+ val func0 = new TableFunc0
+
+ // this case will generate 'timestamp' member field and 'DateFormatter'
+ val result = t
+ .join(func0('c) as('d, 'e))
+ .where(dateFormat(currentTimestamp(), "yyyyMMdd") === 'd)
+ .select('c, 'd, 'e)
+ .toAppendStream[Row]
+
+ result.addSink(new StreamITCase.StringSink[Row])
+ env.execute()
+
+ assertEquals(
+ Seq(),
+ StreamITCase.testResults.sorted
+ )
+ }
+
private def testData(
env: StreamExecutionEnvironment)
: DataStream[(Int, Long, String)] = {