You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/03/05 11:43:04 UTC
[flink] branch release-1.10 updated:
[FLINK-16269][FLINK-16108][table-planner-blink] Fix schema of query and
sink do not match when generic or POJO type is requested (#11236)
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new 9685642 [FLINK-16269][FLINK-16108][table-planner-blink] Fix schema of query and sink do not match when generic or POJO type is requested (#11236)
9685642 is described below
commit 9685642da56554eb5d0292f3ffe193c48329d423
Author: Jark Wu <ja...@apache.org>
AuthorDate: Thu Mar 5 19:40:20 2020 +0800
[FLINK-16269][FLINK-16108][table-planner-blink] Fix schema of query and sink do not match when generic or POJO type is requested (#11236)
This also fixes failure of StreamSQLExample when running in blink planner.
---
.../table/types/inference/TypeTransformations.java | 8 ++
.../transforms/LegacyRawTypeTransformation.java | 46 ++++++++
.../types/inference/TypeTransformationsTest.java | 27 +++++
.../table/planner/codegen/SinkCodeGenerator.scala | 9 +-
.../flink/table/planner/sinks/TableSinkUtils.scala | 122 +++++++++++++--------
.../table/planner/runtime/utils/JavaPojos.java | 81 ++++++++++++++
.../stream/sql/StreamTableEnvironmentITCase.scala | 96 ++++++++++++++++
.../planner/runtime/utils/StreamTestSink.scala | 8 ++
8 files changed, 353 insertions(+), 44 deletions(-)
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TypeTransformations.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TypeTransformations.java
index 0bb8fa6..99e4ff9 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TypeTransformations.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/TypeTransformations.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.inference.transforms.DataTypeConversionClassTransformation;
import org.apache.flink.table.types.inference.transforms.LegacyDecimalTypeTransformation;
+import org.apache.flink.table.types.inference.transforms.LegacyRawTypeTransformation;
import org.apache.flink.table.types.logical.LogicalTypeRoot;
import java.sql.Date;
@@ -59,6 +60,13 @@ public class TypeTransformations {
}
/**
+ * Returns a type transformation that transforms LEGACY('RAW', ...) type to the RAW(..., ?) type.
+ */
+ public static TypeTransformation legacyRawToTypeInfoRaw() {
+ return LegacyRawTypeTransformation.INSTANCE;
+ }
+
+ /**
* Returns a type transformation that transforms data type to nullable data type but keeps
* other information unchanged.
*/
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/transforms/LegacyRawTypeTransformation.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/transforms/LegacyRawTypeTransformation.java
new file mode 100644
index 0000000..c03ff75
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/transforms/LegacyRawTypeTransformation.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.inference.transforms;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.inference.TypeTransformation;
+import org.apache.flink.table.types.logical.LegacyTypeInformationType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+
+/**
+ * This type transformation transforms the LEGACY('RAW', ...) type to the RAW(..., ?) type.
+ */
+public class LegacyRawTypeTransformation implements TypeTransformation {
+
+ public static final TypeTransformation INSTANCE = new LegacyRawTypeTransformation();
+
+ @Override
+ public DataType transform(DataType typeToTransform) {
+ LogicalType logicalType = typeToTransform.getLogicalType();
+ if (logicalType instanceof LegacyTypeInformationType && logicalType.getTypeRoot() == LogicalTypeRoot.RAW) {
+ TypeInformation<?> typeInfo = ((LegacyTypeInformationType<?>) logicalType).getTypeInformation();
+ DataType rawType = DataTypes.RAW(typeInfo).bridgedTo(typeInfo.getTypeClass());
+ return logicalType.isNullable() ? rawType : rawType.notNull();
+ }
+ return typeToTransform;
+ }
+}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/TypeTransformationsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/TypeTransformationsTest.java
index f4b2815..64c0342 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/TypeTransformationsTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/inference/TypeTransformationsTest.java
@@ -18,7 +18,9 @@
package org.apache.flink.table.types.inference;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.utils.DataTypeUtils;
@@ -31,6 +33,7 @@ import java.sql.Time;
import java.sql.Timestamp;
import static org.apache.flink.table.types.inference.TypeTransformations.legacyDecimalToDefaultDecimal;
+import static org.apache.flink.table.types.inference.TypeTransformations.legacyRawToTypeInfoRaw;
import static org.apache.flink.table.types.inference.TypeTransformations.timeToSqlTypes;
import static org.apache.flink.table.types.inference.TypeTransformations.toNullable;
import static org.junit.Assert.assertEquals;
@@ -85,6 +88,26 @@ public class TypeTransformationsTest {
}
@Test
+ public void testLegacyRawToTypeInfoRaw() {
+ DataType dataType = DataTypes.ROW(
+ DataTypes.FIELD("a", DataTypes.STRING()),
+ DataTypes.FIELD("b", DataTypes.DECIMAL(10, 3)),
+ DataTypes.FIELD("c", createLegacyRaw()),
+ DataTypes.FIELD("d", DataTypes.ARRAY(createLegacyRaw()))
+ );
+
+ TypeInformation<TypeTransformationsTest> typeInformation = TypeExtractor.getForClass(TypeTransformationsTest.class);
+ DataType expected = DataTypes.ROW(
+ DataTypes.FIELD("a", DataTypes.STRING()),
+ DataTypes.FIELD("b", DataTypes.DECIMAL(10, 3)),
+ DataTypes.FIELD("c", DataTypes.RAW(typeInformation)),
+ DataTypes.FIELD("d", DataTypes.ARRAY(DataTypes.RAW(typeInformation)))
+ );
+
+ assertEquals(expected, DataTypeUtils.transform(dataType, legacyRawToTypeInfoRaw()));
+ }
+
+ @Test
public void testToNullable() {
DataType dataType = DataTypes.ROW(
DataTypes.FIELD("a", DataTypes.STRING().notNull()),
@@ -110,4 +133,8 @@ public class TypeTransformationsTest {
private static DataType createLegacyDecimal() {
return TypeConversions.fromLegacyInfoToDataType(Types.BIG_DEC);
}
+
+ private static DataType createLegacyRaw() {
+ return TypeConversions.fromLegacyInfoToDataType(Types.GENERIC(TypeTransformationsTest.class));
+ }
}
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/SinkCodeGenerator.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/SinkCodeGenerator.scala
index 9a807f1..2cc9a65 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/SinkCodeGenerator.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/SinkCodeGenerator.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.planner.codegen
import org.apache.flink.api.common.ExecutionConfig
import org.apache.flink.api.common.typeinfo.{TypeInformation, Types}
+import org.apache.flink.api.common.typeutils.CompositeType
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
import org.apache.flink.api.java.typeutils.{PojoTypeInfo, TupleTypeInfo}
import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase
@@ -33,6 +34,8 @@ import org.apache.flink.table.planner.codegen.OperatorCodeGenerator.generateColl
import org.apache.flink.table.planner.sinks.TableSinkUtils
import org.apache.flink.table.runtime.operators.CodeGenOperatorFactory
import org.apache.flink.table.runtime.types.TypeInfoDataTypeConverter.fromDataTypeToTypeInfo
+import org.apache.flink.table.runtime.types.TypeInfoLogicalTypeConverter.fromTypeInfoToLogicalType
+import org.apache.flink.table.runtime.typeutils.BaseRowTypeInfo
import org.apache.flink.table.sinks.TableSink
import org.apache.flink.table.types.logical.RowType
@@ -83,8 +86,12 @@ object SinkCodeGenerator {
inputRowType,
inputTerm,
inputFieldMapping = Option(mapping))
+ val outputRowType = RowType.of(
+ (0 until pojo.getArity)
+ .map(pojo.getTypeAt)
+ .map(fromTypeInfoToLogicalType): _*)
val conversion = resultGenerator.generateConverterResultExpression(
- inputRowType,
+ outputRowType,
classOf[GenericRow])
afterIndexModify = CodeGenUtils.newName("afterIndexModify")
s"""
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/sinks/TableSinkUtils.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/sinks/TableSinkUtils.scala
index 63e6bbe..90955ee 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/sinks/TableSinkUtils.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/sinks/TableSinkUtils.scala
@@ -19,7 +19,7 @@
package org.apache.flink.table.planner.sinks
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
-import org.apache.flink.api.java.typeutils.{GenericTypeInfo, TupleTypeInfo}
+import org.apache.flink.api.java.typeutils.{GenericTypeInfo, PojoTypeInfo, TupleTypeInfo}
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.table.api._
import org.apache.flink.table.catalog.{CatalogTable, ObjectIdentifier}
@@ -27,17 +27,17 @@ import org.apache.flink.table.dataformat.BaseRow
import org.apache.flink.table.operations.CatalogSinkModifyOperation
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
import org.apache.flink.table.planner.plan.utils.RelOptUtils
+import org.apache.flink.table.runtime.types.TypeInfoDataTypeConverter.fromDataTypeToTypeInfo
import org.apache.flink.table.runtime.typeutils.BaseRowTypeInfo
import org.apache.flink.table.sinks._
import org.apache.flink.table.types.DataType
-import org.apache.flink.table.types.inference.TypeTransformations.{legacyDecimalToDefaultDecimal, toNullable}
+import org.apache.flink.table.types.inference.TypeTransformations.{legacyDecimalToDefaultDecimal, toNullable, legacyRawToTypeInfoRaw}
import org.apache.flink.table.types.logical.utils.{LogicalTypeCasts, LogicalTypeChecks}
-import org.apache.flink.table.types.logical.{LegacyTypeInformationType, LogicalType, RowType}
+import org.apache.flink.table.types.logical.{LegacyTypeInformationType, LogicalType, RowType, TypeInformationRawType}
import org.apache.flink.table.types.utils.DataTypeUtils
import org.apache.flink.table.types.utils.TypeConversions.{fromLegacyInfoToDataType, fromLogicalToDataType}
import org.apache.flink.table.utils.{TableSchemaUtils, TypeMappingUtils}
import org.apache.flink.types.Row
-
import org.apache.calcite.rel.RelNode
import _root_.scala.collection.JavaConversions._
@@ -64,7 +64,7 @@ object TableSinkUtils {
val queryLogicalType = FlinkTypeFactory.toLogicalRowType(query.getRowType)
val sinkLogicalType = DataTypeUtils
// we recognize legacy decimal is the same to default decimal
- .transform(sinkSchema.toRowDataType, legacyDecimalToDefaultDecimal)
+ .transform(sinkSchema.toRowDataType, legacyDecimalToDefaultDecimal, legacyRawToTypeInfoRaw)
.getLogicalType
.asInstanceOf[RowType]
if (LogicalTypeCasts.supportsImplicitCast(queryLogicalType, sinkLogicalType)) {
@@ -189,7 +189,12 @@ object TableSinkUtils {
queryLogicalType,
withChangeFlag)
if (LogicalTypeChecks.isCompositeType(requestedOutputType.getLogicalType)) {
- DataTypeUtils.expandCompositeTypeToSchema(requestedOutputType)
+ // if the requested output type is POJO, then we should ignore the POJO fields order,
+ // and infer the sink schema via field names, see expandPojoTypeToSchema().
+ fromDataTypeToTypeInfo(requestedOutputType) match {
+ case pj: PojoTypeInfo[_] => expandPojoTypeToSchema(pj, queryLogicalType)
+ case _ => DataTypeUtils.expandCompositeTypeToSchema(requestedOutputType)
+ }
} else {
// atomic type
TableSchema.builder().field("f0", requestedOutputType).build()
@@ -197,6 +202,35 @@ object TableSinkUtils {
}
/**
+ * Expands a [[PojoTypeInfo]] to a corresponding [[TableSchema]].
+ * This is a special handling for [[PojoTypeInfo]], because fields of [[PojoTypeInfo]] is not
+ * in the defined order but the alphabet order. In order to match the query schema, we should
+ * reorder the Pojo schema.
+ */
+ private def expandPojoTypeToSchema(
+ pojo: PojoTypeInfo[_],
+ queryLogicalType: RowType): TableSchema = {
+ val fieldNames = queryLogicalType.getFieldNames
+ // reorder pojo fields according to the query schema
+ val reorderedFields = fieldNames.map(name => {
+ val index = pojo.getFieldIndex(name)
+ if (index < 0) {
+ throw new TableException(s"$name is not found in ${pojo.toString}")
+ }
+ val fieldTypeInfo = pojo.getPojoFieldAt(index).getTypeInformation
+ val fieldDataType = fieldTypeInfo match {
+ case nestedPojo: PojoTypeInfo[_] =>
+ val nestedLogicalType = queryLogicalType.getFields()(index).getType.asInstanceOf[RowType]
+ expandPojoTypeToSchema(nestedPojo, nestedLogicalType).toRowDataType
+ case _ =>
+ fromLegacyInfoToDataType(fieldTypeInfo)
+ }
+ DataTypes.FIELD(name, fieldDataType)
+ })
+ DataTypeUtils.expandCompositeTypeToSchema(DataTypes.ROW(reorderedFields: _*))
+ }
+
+ /**
* Inferences the physical data type of [[TableSink]], the physical data type ignores
* the change flag field.
*
@@ -209,45 +243,47 @@ object TableSinkUtils {
consumedDataType: DataType,
queryLogicalType: RowType,
withChangeFlag: Boolean): DataType = {
- consumedDataType.getLogicalType match {
- case lt: LegacyTypeInformationType[_] =>
- val requestedTypeInfo = if (withChangeFlag) {
- lt.getTypeInformation match {
- // Scala tuple
- case t: CaseClassTypeInfo[_]
- if t.getTypeClass == classOf[(_, _)] && t.getTypeAt(0) == Types.BOOLEAN =>
- t.getTypeAt[Any](1)
- // Java tuple
- case t: TupleTypeInfo[_]
- if t.getTypeClass == classOf[JTuple2[_, _]] && t.getTypeAt(0) == Types.BOOLEAN =>
- t.getTypeAt[Any](1)
- case _ => throw new TableException(
- "Don't support " + consumedDataType + " conversion for the retract sink")
- }
- } else {
- lt.getTypeInformation
- }
- // The tpe may been inferred by invoking [[TypeExtractor.createTypeInfo]] based the
- // class of the resulting type. For example, converts the given [[Table]] into
- // an append [[DataStream]]. If the class is Row, then the return type only is
- // [[GenericTypeInfo[Row]]. So it should convert to the [[RowTypeInfo]] in order
- // to better serialize performance.
- requestedTypeInfo match {
- case gt: GenericTypeInfo[Row] if gt.getTypeClass == classOf[Row] =>
- fromLogicalToDataType(queryLogicalType).bridgedTo(classOf[Row])
- case gt: GenericTypeInfo[BaseRow] if gt.getTypeClass == classOf[BaseRow] =>
- fromLogicalToDataType(queryLogicalType).bridgedTo(classOf[BaseRow])
- case bt: BaseRowTypeInfo =>
- val fields = bt.getFieldNames.zip(bt.getLogicalTypes).map { case (n, t) =>
- DataTypes.FIELD(n, fromLogicalToDataType(t))
- }
- DataTypes.ROW(fields: _*).bridgedTo(classOf[BaseRow])
- case _ =>
- fromLegacyInfoToDataType(requestedTypeInfo)
- }
+ val consumedTypeInfo = consumedDataType.getLogicalType match {
+ case lt: LegacyTypeInformationType[_] => Some(lt.getTypeInformation)
+ case _ => None
+ }
+ if (consumedTypeInfo.isEmpty) {
+ return consumedDataType
+ }
+ val requestedTypeInfo = if (withChangeFlag) {
+ consumedTypeInfo.get match {
+ // Scala tuple
+ case t: CaseClassTypeInfo[_]
+ if t.getTypeClass == classOf[(_, _)] && t.getTypeAt(0) == Types.BOOLEAN =>
+ t.getTypeAt[Any](1)
+ // Java tuple
+ case t: TupleTypeInfo[_]
+ if t.getTypeClass == classOf[JTuple2[_, _]] && t.getTypeAt(0) == Types.BOOLEAN =>
+ t.getTypeAt[Any](1)
+ case _ => throw new TableException(
+ "Don't support " + consumedDataType + " conversion for the retract sink")
+ }
+ } else {
+ consumedTypeInfo.get
+ }
+ // The tpe may been inferred by invoking [[TypeExtractor.createTypeInfo]] based the
+ // class of the resulting type. For example, converts the given [[Table]] into
+ // an append [[DataStream]]. If the class is Row, then the return type only is
+ // [[GenericTypeInfo[Row]]. So it should convert to the [[RowTypeInfo]] in order
+ // to better serialize performance.
+ requestedTypeInfo match {
+ case gt: GenericTypeInfo[Row] if gt.getTypeClass == classOf[Row] =>
+ fromLogicalToDataType(queryLogicalType).bridgedTo(classOf[Row])
+ case gt: GenericTypeInfo[BaseRow] if gt.getTypeClass == classOf[BaseRow] =>
+ fromLogicalToDataType(queryLogicalType).bridgedTo(classOf[BaseRow])
+ case bt: BaseRowTypeInfo =>
+ val fields = bt.getFieldNames.zip(bt.getLogicalTypes).map { case (n, t) =>
+ DataTypes.FIELD(n, fromLogicalToDataType(t))
+ }
+ DataTypes.ROW(fields: _*).bridgedTo(classOf[BaseRow])
case _ =>
- consumedDataType
+ fromLegacyInfoToDataType(requestedTypeInfo)
}
}
diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/JavaPojos.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/JavaPojos.java
index d8c4537..a64fe05 100644
--- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/JavaPojos.java
+++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/JavaPojos.java
@@ -20,6 +20,8 @@ package org.apache.flink.table.planner.runtime.utils;
import java.io.Serializable;
import java.sql.Timestamp;
+import java.util.Map;
+import java.util.TreeMap;
/**
* POJOs for Table API testing.
@@ -42,4 +44,83 @@ public class JavaPojos {
'}';
}
}
+
+ /**
+ * Nested POJO.
+ */
+ public static class Order {
+ public Long user;
+ public ProductItem product;
+ public int amount;
+
+ public Order() {
+ }
+
+ public Order(Long user, ProductItem product, int amount) {
+ this.user = user;
+ this.product = product;
+ this.amount = amount;
+ }
+
+ @Override
+ public String toString() {
+ return "Order{" +
+ "user=" + user +
+ ", product='" + product + '\'' +
+ ", amount=" + amount +
+ '}';
+ }
+ }
+
+ /**
+ * Simple POJO.
+ */
+ public static class ProductItem {
+ public String name;
+ public Long id;
+
+ public ProductItem() {
+ }
+
+ public ProductItem(String name, Long id) {
+ this.name = name;
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ return "Product{" +
+ "name='" + name + '\'' +
+ ", id=" + id +
+ '}';
+ }
+ }
+
+ /**
+ * POJO with a RAW type.
+ */
+ public static class Device {
+ public Long deviceId;
+ public String deviceName;
+ // raw type
+ public TreeMap<String, Long> metrics;
+
+ public Device() {
+ }
+
+ public Device(Long deviceId, String deviceName, Map<String, Long> metrics) {
+ this.deviceId = deviceId;
+ this.deviceName = deviceName;
+ this.metrics = new TreeMap<>(metrics);
+ }
+
+ @Override
+ public String toString() {
+ return "Device{" +
+ "deviceId=" + deviceId +
+ ", deviceName='" + deviceName + '\'' +
+ ", metrics=" + metrics +
+ '}';
+ }
+ }
}
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala
new file mode 100644
index 0000000..d26c864
--- /dev/null
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/StreamTableEnvironmentITCase.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.runtime.stream.sql
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.DataStream
+import org.apache.flink.table.api.scala._
+import org.apache.flink.table.planner.runtime.utils.JavaPojos.{Device, Order, ProductItem}
+import org.apache.flink.table.planner.runtime.utils.{StreamingTestBase, StringSink}
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+import java.util.Collections
+
+/**
+ * Integration tests for methods on [[org.apache.flink.table.api.scala.StreamTableEnvironment]].
+ */
+class StreamTableEnvironmentITCase extends StreamingTestBase {
+
+ @Test
+ def testToAppendStreamWithPojoType(): Unit = {
+ val orderA = env.fromCollection(Seq(
+ new Order(1L, new ProductItem("beer", 10L), 3),
+ new Order(1L, new ProductItem("diaper", 11L), 4),
+ new Order(3L, new ProductItem("rubber", 12L), 2)))
+
+ val orderB: DataStream[Order] = env.fromCollection(Seq(
+ new Order(2L, new ProductItem("pen", 13L), 3),
+ new Order(2L, new ProductItem("rubber", 12L), 3),
+ new Order(4L, new ProductItem("beer", 10L), 1)))
+
+ // convert DataStream to Table
+ val tableA = tEnv.fromDataStream(orderA, 'user, 'product, 'amount)
+ // register DataStream as Table
+ tEnv.createTemporaryView("OrderB", orderB, 'user, 'product, 'amount)
+
+ // union the two tables
+ val result = tEnv.sqlQuery(
+ s"""
+ |SELECT * FROM $tableA WHERE amount > 2
+ |UNION ALL
+ |SELECT * FROM OrderB WHERE amount < 2
+ """.stripMargin)
+
+ val sink = new StringSink[Order]()
+ result.toAppendStream[Order].addSink(sink)
+
+ env.execute()
+
+ val expected = List(
+ "Order{user=1, product='Product{name='beer', id=10}', amount=3}",
+ "Order{user=1, product='Product{name='diaper', id=11}', amount=4}",
+ "Order{user=4, product='Product{name='beer', id=10}', amount=1}")
+ assertEquals(expected.sorted, sink.getResults.sorted)
+ }
+
+ @Test
+ def testToAppendStreamWithRawType(): Unit = {
+ val devices = env.fromCollection(Seq(
+ new Device(1L, "device1", Collections.singletonMap("A", 10)),
+ new Device(2L, "device2", Collections.emptyMap()),
+ new Device(3L, "device3", Collections.singletonMap("B", 20))
+ ))
+
+ // register DataStream as Table
+ tEnv.createTemporaryView("devices", devices,'deviceId, 'deviceName, 'metrics)
+
+ val result = tEnv.sqlQuery("SELECT * FROM devices WHERE deviceId >= 2")
+ val sink = new StringSink[Device]()
+ result.toAppendStream[Device].addSink(sink)
+
+ env.execute()
+
+ val expected = List(
+ "Device{deviceId=2, deviceName='device2', metrics={}}",
+ "Device{deviceId=3, deviceName='device3', metrics={B=20}}")
+ assertEquals(expected.sorted, sink.getResults.sorted)
+ }
+}
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTestSink.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTestSink.scala
index ac54eca..b3b1db1 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTestSink.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/utils/StreamTestSink.scala
@@ -127,6 +127,14 @@ abstract class AbstractExactlyOnceSink[T] extends RichSinkFunction[T] with Check
}
}
+final class StringSink[T] extends AbstractExactlyOnceSink[T]() {
+ override def invoke(value: T) {
+ localResults += value.toString
+ }
+
+ override def getResults: List[String] = super.getResults
+}
+
final class TestingAppendBaseRowSink(
rowTypeInfo: BaseRowTypeInfo, tz: TimeZone)
extends AbstractExactlyOnceSink[BaseRow] {