You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/01/02 12:30:46 UTC
[flink] branch release-1.10 updated: [FLINK-15334][table-api] Fix
physical schema mapping in TableFormatFactoryBase to support define
orderless computed column (#10693)
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new a1fe7ad [FLINK-15334][table-api] Fix physical schema mapping in TableFormatFactoryBase to support define orderless computed column (#10693)
a1fe7ad is described below
commit a1fe7adfd1f7b1807d611614ae88838bb178023f
Author: Leonard Xu <xb...@163.com>
AuthorDate: Thu Jan 2 20:29:57 2020 +0800
[FLINK-15334][table-api] Fix physical schema mapping in TableFormatFactoryBase to support define orderless computed column (#10693)
---
.../connectors/kafka/KafkaTableTestBase.java | 2 ++
.../flink/table/descriptors/SchemaValidator.java | 31 +++++++++++--------
.../table/factories/TableFormatFactoryBase.java | 26 ++++++++--------
.../table/descriptors/SchemaValidatorTest.scala | 36 ++++++++++++++--------
4 files changed, 57 insertions(+), 38 deletions(-)
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
index 26837ad..c6558ac 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
@@ -72,6 +72,7 @@ public abstract class KafkaTableTestBase extends KafkaTestBase {
// TODO: use DDL to register Kafka once FLINK-15282 is fixed.
// we have to register into Catalog manually because it will use Calcite's ParameterScope
TableSchema schema = TableSchema.builder()
+ .field("computed-price", DataTypes.DECIMAL(38, 18), "price + 1.0")
.field("price", DataTypes.DECIMAL(38, 18))
.field("currency", DataTypes.STRING())
.field("log_ts", DataTypes.TIMESTAMP(3))
@@ -101,6 +102,7 @@ public abstract class KafkaTableTestBase extends KafkaTestBase {
// TODO: use the following DDL instead of the preceding code to register Kafka
// String ddl = "CREATE TABLE kafka (\n" +
+// " computed-price as price + 1.0,\n" +
// " price DECIMAL(38, 18),\n" +
// " currency STRING,\n" +
// " log_ts TIMESTAMP(3),\n" +
diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
index d3c57cf..2bb1dc0 100644
--- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
+++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.api.TableColumn;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.api.ValidationException;
@@ -29,7 +30,7 @@ import org.apache.flink.table.factories.TableFormatFactory;
import org.apache.flink.table.sources.RowtimeAttributeDescriptor;
import org.apache.flink.table.sources.tsextractors.TimestampExtractor;
import org.apache.flink.table.sources.wmstrategies.WatermarkStrategy;
-import org.apache.flink.table.utils.TableSchemaUtils;
+import org.apache.flink.table.types.DataType;
import java.util.ArrayList;
import java.util.Arrays;
@@ -212,12 +213,16 @@ public class SchemaValidator implements DescriptorValidator {
@Deprecated
public static TableSchema deriveTableSinkSchema(DescriptorProperties properties) {
TableSchema.Builder builder = TableSchema.builder();
-
- TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
-
- for (int i = 0; i < schema.getFieldCount(); i++) {
- TypeInformation t = schema.getFieldTypes()[i];
- String n = schema.getFieldNames()[i];
+ TableSchema tableSchema = properties.getTableSchema(SCHEMA);
+ for (int i = 0; i < tableSchema.getFieldCount(); i++) {
+ final TableColumn tableColumn = tableSchema.getTableColumns().get(i);
+ final String fieldName = tableColumn.getName();
+ final DataType dataType = tableColumn.getType();
+ boolean isGeneratedColumn = tableColumn.isGenerated();
+ if (isGeneratedColumn) {
+ // skip generated column
+ continue;
+ }
boolean isProctime = properties
.getOptionalBoolean(SCHEMA + "." + i + "." + SCHEMA_PROCTIME)
.orElse(false);
@@ -225,23 +230,23 @@ public class SchemaValidator implements DescriptorValidator {
boolean isRowtime = properties.containsKey(tsType);
if (!isProctime && !isRowtime) {
// check for a aliasing
- String fieldName = properties.getOptionalString(SCHEMA + "." + i + "." + SCHEMA_FROM)
- .orElse(n);
- builder.field(fieldName, t);
+ String aliasName = properties.getOptionalString(SCHEMA + "." + i + "." + SCHEMA_FROM)
+ .orElse(fieldName);
+ builder.field(aliasName, dataType);
}
// only use the rowtime attribute if it references a field
else if (isRowtime) {
switch (properties.getString(tsType)) {
case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD:
String field = properties.getString(SCHEMA + "." + i + "." + ROWTIME_TIMESTAMPS_FROM);
- builder.field(field, t);
+ builder.field(field, dataType);
break;
// other timestamp strategies require a reverse timestamp extractor to
// insert the timestamp into the output
default:
throw new TableException(format("Unsupported rowtime type '%s' for sink" +
" table schema. Currently only '%s' is supported for table sinks.",
- t, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD));
+ dataType, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD));
}
}
}
@@ -261,7 +266,7 @@ public class SchemaValidator implements DescriptorValidator {
Map<String, String> mapping = new HashMap<>();
- TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
+ TableSchema schema = properties.getTableSchema(SCHEMA);
List<String> columnNames = new ArrayList<>();
inputType.ifPresent(t -> columnNames.addAll(Arrays.asList(((CompositeType) t).getFieldNames())));
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
index 8ff62c8..fdae4d6 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
@@ -19,11 +19,11 @@
package org.apache.flink.table.factories;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.table.api.TableColumn;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.descriptors.FormatDescriptorValidator;
import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.utils.TableSchemaUtils;
import java.util.ArrayList;
import java.util.Collections;
@@ -154,32 +154,34 @@ public abstract class TableFormatFactoryBase<T> implements TableFormatFactory<T>
final TableSchema.Builder builder = TableSchema.builder();
- final TableSchema baseSchema = TableSchemaUtils.getPhysicalSchema(
- descriptorProperties.getTableSchema(SCHEMA));
- for (int i = 0; i < baseSchema.getFieldCount(); i++) {
- final String fieldName = baseSchema.getFieldNames()[i];
- final DataType fieldType = baseSchema.getFieldDataTypes()[i];
-
+ final TableSchema tableSchema = descriptorProperties.getTableSchema(SCHEMA);
+ for (int i = 0; i < tableSchema.getFieldCount(); i++) {
+ final TableColumn tableColumn = tableSchema.getTableColumns().get(i);
+ final String fieldName = tableColumn.getName();
+ final DataType dataType = tableColumn.getType();
+ final boolean isGeneratedColumn = tableColumn.isGenerated();
+ if (isGeneratedColumn) {
+ //skip generated column
+ continue;
+ }
final boolean isProctime = descriptorProperties
.getOptionalBoolean(SCHEMA + '.' + i + '.' + SCHEMA_PROCTIME)
.orElse(false);
final String timestampKey = SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_TYPE;
final boolean isRowtime = descriptorProperties.containsKey(timestampKey);
- boolean isGeneratedColumn = properties.containsKey(SCHEMA + "." + i + "." + TABLE_SCHEMA_EXPR);
-
- if (!isProctime && !isRowtime && !isGeneratedColumn) {
+ if (!isProctime && !isRowtime) {
// check for aliasing
final String aliasName = descriptorProperties
.getOptionalString(SCHEMA + '.' + i + '.' + SCHEMA_FROM)
.orElse(fieldName);
- builder.field(aliasName, fieldType);
+ builder.field(aliasName, dataType);
}
// only use the rowtime attribute if it references a field
else if (isRowtime &&
descriptorProperties.isValue(timestampKey, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD)) {
final String aliasName = descriptorProperties
.getString(SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_FROM);
- builder.field(aliasName, fieldType);
+ builder.field(aliasName, dataType);
}
}
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
index ce04e5c..c946885 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
@@ -168,25 +168,35 @@ class SchemaValidatorTest {
@Test
def testSchemaWithGeneratedColumnAndWatermark(): Unit = {
- val descriptor = new Schema()
- .field("f1", DataTypes.STRING)
- .field("f2", DataTypes.INT)
- .field("f3", DataTypes.TIMESTAMP(3))
-
val properties = new DescriptorProperties()
- properties.putProperties(descriptor.toProperties)
- properties.putString("schema.3.name", "generated-column")
- properties.putString("schema.3.data-type", DataTypes.INT.toString)
- properties.putString("schema.3.expr", "f2 + 1")
- properties.putString("schema.watermark.0.rowtime", "f3")
- properties.putString("schema.watermark.0.strategy.expr", "f3 - INTERVAL '5' SECOND")
+ properties.putString("schema.0.name", "f1")
+ properties.putString("schema.0.data-type", DataTypes.STRING().toString)
+ properties.putString("schema.1.name", "computed-column")
+ properties.putString("schema.1.data-type", DataTypes.INT().toString)
+ properties.putString("schema.1.expr", "f5 + 1")
+ properties.putString("schema.2.name", "f2")
+ properties.putString("schema.2.data-type", DataTypes.INT().toString)
+ properties.putString("schema.3.name", "f3")
+ properties.putString("schema.3.data-type", DataTypes.TIMESTAMP(3).toString)
+ properties.putString("schema.4.name", "row-time")
+ properties.putString("schema.4.data-type", DataTypes.TIMESTAMP(3).toString)
+ properties.putString("schema.4.rowtime.timestamps.type", "from-field")
+ properties.putString("schema.4.rowtime.timestamps.from", "f4")
+ properties.putString("schema.4.rowtime.watermarks.type", "periodic-bounded")
+ properties.putString("schema.4.rowtime.watermarks.delay", "5000")
+ properties.putString("schema.5.name", "f5")
+ properties.putString("schema.5.data-type", DataTypes.INT().toString)
+ properties.putString("schema.watermark.0.rowtime", "row-time")
+ properties.putString("schema.watermark.0.strategy.expr", "row-time - INTERVAL '5' SECOND")
properties.putString("schema.watermark.0.strategy.data-type", DataTypes.TIMESTAMP(3).toString)
new SchemaValidator(true, true, false).validate(properties)
val expectd = TableSchema.builder()
- .field("f1", DataTypes.STRING)
- .field("f2", DataTypes.INT)
+ .field("f1", DataTypes.STRING())
+ .field("f2", DataTypes.INT())
.field("f3", DataTypes.TIMESTAMP(3))
+ .field("f4", DataTypes.TIMESTAMP(3))
+ .field("f5", DataTypes.INT())
.build()
val schema = SchemaValidator.deriveTableSinkSchema(properties)
assertEquals(expectd, schema)