You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/01/02 12:30:46 UTC

[flink] branch release-1.10 updated: [FLINK-15334][table-api] Fix physical schema mapping in TableFormatFactoryBase to support define orderless computed column (#10693)

This is an automated email from the ASF dual-hosted git repository.

jark pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.10 by this push:
     new a1fe7ad  [FLINK-15334][table-api] Fix physical schema mapping in TableFormatFactoryBase to support define orderless computed column (#10693)
a1fe7ad is described below

commit a1fe7adfd1f7b1807d611614ae88838bb178023f
Author: Leonard Xu <xb...@163.com>
AuthorDate: Thu Jan 2 20:29:57 2020 +0800

    [FLINK-15334][table-api] Fix physical schema mapping in TableFormatFactoryBase to support define orderless computed column (#10693)
---
 .../connectors/kafka/KafkaTableTestBase.java       |  2 ++
 .../flink/table/descriptors/SchemaValidator.java   | 31 +++++++++++--------
 .../table/factories/TableFormatFactoryBase.java    | 26 ++++++++--------
 .../table/descriptors/SchemaValidatorTest.scala    | 36 ++++++++++++++--------
 4 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
index 26837ad..c6558ac 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableTestBase.java
@@ -72,6 +72,7 @@ public abstract class KafkaTableTestBase extends KafkaTestBase {
 		// TODO: use DDL to register Kafka once FLINK-15282 is fixed.
 		//  we have to register into Catalog manually because it will use Calcite's ParameterScope
 		TableSchema schema = TableSchema.builder()
+			.field("computed-price", DataTypes.DECIMAL(38, 18), "price + 1.0")
 			.field("price", DataTypes.DECIMAL(38, 18))
 			.field("currency", DataTypes.STRING())
 			.field("log_ts", DataTypes.TIMESTAMP(3))
@@ -101,6 +102,7 @@ public abstract class KafkaTableTestBase extends KafkaTestBase {
 
 		// TODO: use the following DDL instead of the preceding code to register Kafka
 //		String ddl = "CREATE TABLE kafka (\n" +
+//			"  computed-price as price + 1.0,\n" +
 //			"  price DECIMAL(38, 18),\n" +
 //			"  currency STRING,\n" +
 //			"  log_ts TIMESTAMP(3),\n" +
diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
index d3c57cf..2bb1dc0 100644
--- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
+++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/descriptors/SchemaValidator.java
@@ -22,6 +22,7 @@ import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.CompositeType;
 import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.api.TableColumn;
 import org.apache.flink.table.api.TableException;
 import org.apache.flink.table.api.TableSchema;
 import org.apache.flink.table.api.ValidationException;
@@ -29,7 +30,7 @@ import org.apache.flink.table.factories.TableFormatFactory;
 import org.apache.flink.table.sources.RowtimeAttributeDescriptor;
 import org.apache.flink.table.sources.tsextractors.TimestampExtractor;
 import org.apache.flink.table.sources.wmstrategies.WatermarkStrategy;
-import org.apache.flink.table.utils.TableSchemaUtils;
+import org.apache.flink.table.types.DataType;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -212,12 +213,16 @@ public class SchemaValidator implements DescriptorValidator {
 	@Deprecated
 	public static TableSchema deriveTableSinkSchema(DescriptorProperties properties) {
 		TableSchema.Builder builder = TableSchema.builder();
-
-		TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
-
-		for (int i = 0; i < schema.getFieldCount(); i++) {
-			TypeInformation t = schema.getFieldTypes()[i];
-			String n = schema.getFieldNames()[i];
+		TableSchema tableSchema = properties.getTableSchema(SCHEMA);
+		for (int i = 0; i < tableSchema.getFieldCount(); i++) {
+			final TableColumn tableColumn = tableSchema.getTableColumns().get(i);
+			final String fieldName = tableColumn.getName();
+			final DataType dataType = tableColumn.getType();
+			boolean isGeneratedColumn = tableColumn.isGenerated();
+			if (isGeneratedColumn) {
+				// skip generated column
+				continue;
+			}
 			boolean isProctime = properties
 					.getOptionalBoolean(SCHEMA + "." + i + "." + SCHEMA_PROCTIME)
 					.orElse(false);
@@ -225,23 +230,23 @@ public class SchemaValidator implements DescriptorValidator {
 			boolean isRowtime = properties.containsKey(tsType);
 			if (!isProctime && !isRowtime) {
 				// check for a aliasing
-				String fieldName = properties.getOptionalString(SCHEMA + "." + i + "." + SCHEMA_FROM)
-						.orElse(n);
-				builder.field(fieldName, t);
+				String aliasName = properties.getOptionalString(SCHEMA + "." + i + "." + SCHEMA_FROM)
+						.orElse(fieldName);
+				builder.field(aliasName, dataType);
 			}
 			// only use the rowtime attribute if it references a field
 			else if (isRowtime) {
 				switch (properties.getString(tsType)) {
 					case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD:
 						String field = properties.getString(SCHEMA + "." + i + "." + ROWTIME_TIMESTAMPS_FROM);
-						builder.field(field, t);
+						builder.field(field, dataType);
 						break;
 					// other timestamp strategies require a reverse timestamp extractor to
 					// insert the timestamp into the output
 					default:
 						throw new TableException(format("Unsupported rowtime type '%s' for sink" +
 								" table schema. Currently only '%s' is supported for table sinks.",
-								t, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD));
+							dataType, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD));
 				}
 			}
 		}
@@ -261,7 +266,7 @@ public class SchemaValidator implements DescriptorValidator {
 
 		Map<String, String> mapping = new HashMap<>();
 
-		TableSchema schema = TableSchemaUtils.getPhysicalSchema(properties.getTableSchema(SCHEMA));
+		TableSchema schema = properties.getTableSchema(SCHEMA);
 
 		List<String> columnNames = new ArrayList<>();
 		inputType.ifPresent(t -> columnNames.addAll(Arrays.asList(((CompositeType) t).getFieldNames())));
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
index 8ff62c8..fdae4d6 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
@@ -19,11 +19,11 @@
 package org.apache.flink.table.factories;
 
 import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.table.api.TableColumn;
 import org.apache.flink.table.api.TableSchema;
 import org.apache.flink.table.descriptors.DescriptorProperties;
 import org.apache.flink.table.descriptors.FormatDescriptorValidator;
 import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.utils.TableSchemaUtils;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -154,32 +154,34 @@ public abstract class TableFormatFactoryBase<T> implements TableFormatFactory<T>
 
 		final TableSchema.Builder builder = TableSchema.builder();
 
-		final TableSchema baseSchema = TableSchemaUtils.getPhysicalSchema(
-			descriptorProperties.getTableSchema(SCHEMA));
-		for (int i = 0; i < baseSchema.getFieldCount(); i++) {
-			final String fieldName = baseSchema.getFieldNames()[i];
-			final DataType fieldType = baseSchema.getFieldDataTypes()[i];
-
+		final TableSchema tableSchema = descriptorProperties.getTableSchema(SCHEMA);
+		for (int i = 0; i < tableSchema.getFieldCount(); i++) {
+			final TableColumn tableColumn = tableSchema.getTableColumns().get(i);
+			final String fieldName = tableColumn.getName();
+			final DataType dataType = tableColumn.getType();
+			final boolean isGeneratedColumn = tableColumn.isGenerated();
+			if (isGeneratedColumn) {
+				//skip generated column
+				continue;
+			}
 			final boolean isProctime = descriptorProperties
 				.getOptionalBoolean(SCHEMA + '.' + i + '.' + SCHEMA_PROCTIME)
 				.orElse(false);
 			final String timestampKey = SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_TYPE;
 			final boolean isRowtime = descriptorProperties.containsKey(timestampKey);
-			boolean isGeneratedColumn = properties.containsKey(SCHEMA + "." + i + "." + TABLE_SCHEMA_EXPR);
-
-			if (!isProctime && !isRowtime && !isGeneratedColumn) {
+			if (!isProctime && !isRowtime) {
 				// check for aliasing
 				final String aliasName = descriptorProperties
 					.getOptionalString(SCHEMA + '.' + i + '.' + SCHEMA_FROM)
 					.orElse(fieldName);
-				builder.field(aliasName, fieldType);
+				builder.field(aliasName, dataType);
 			}
 			// only use the rowtime attribute if it references a field
 			else if (isRowtime &&
 					descriptorProperties.isValue(timestampKey, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD)) {
 				final String aliasName = descriptorProperties
 					.getString(SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_FROM);
-				builder.field(aliasName, fieldType);
+				builder.field(aliasName, dataType);
 			}
 		}
 
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
index ce04e5c..c946885 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
@@ -168,25 +168,35 @@ class SchemaValidatorTest {
 
   @Test
   def testSchemaWithGeneratedColumnAndWatermark(): Unit = {
-    val descriptor = new Schema()
-      .field("f1", DataTypes.STRING)
-      .field("f2", DataTypes.INT)
-      .field("f3", DataTypes.TIMESTAMP(3))
-
     val properties = new DescriptorProperties()
-    properties.putProperties(descriptor.toProperties)
-    properties.putString("schema.3.name", "generated-column")
-    properties.putString("schema.3.data-type", DataTypes.INT.toString)
-    properties.putString("schema.3.expr", "f2 + 1")
-    properties.putString("schema.watermark.0.rowtime", "f3")
-    properties.putString("schema.watermark.0.strategy.expr", "f3 - INTERVAL '5' SECOND")
+    properties.putString("schema.0.name", "f1")
+    properties.putString("schema.0.data-type", DataTypes.STRING().toString)
+    properties.putString("schema.1.name", "computed-column")
+    properties.putString("schema.1.data-type", DataTypes.INT().toString)
+    properties.putString("schema.1.expr", "f5 + 1")
+    properties.putString("schema.2.name", "f2")
+    properties.putString("schema.2.data-type", DataTypes.INT().toString)
+    properties.putString("schema.3.name", "f3")
+    properties.putString("schema.3.data-type", DataTypes.TIMESTAMP(3).toString)
+    properties.putString("schema.4.name", "row-time")
+    properties.putString("schema.4.data-type", DataTypes.TIMESTAMP(3).toString)
+    properties.putString("schema.4.rowtime.timestamps.type", "from-field")
+    properties.putString("schema.4.rowtime.timestamps.from", "f4")
+    properties.putString("schema.4.rowtime.watermarks.type", "periodic-bounded")
+    properties.putString("schema.4.rowtime.watermarks.delay", "5000")
+    properties.putString("schema.5.name", "f5")
+    properties.putString("schema.5.data-type", DataTypes.INT().toString)
+    properties.putString("schema.watermark.0.rowtime", "row-time")
+    properties.putString("schema.watermark.0.strategy.expr", "row-time - INTERVAL '5' SECOND")
     properties.putString("schema.watermark.0.strategy.data-type", DataTypes.TIMESTAMP(3).toString)
 
     new SchemaValidator(true, true, false).validate(properties)
     val expectd = TableSchema.builder()
-      .field("f1", DataTypes.STRING)
-      .field("f2", DataTypes.INT)
+      .field("f1", DataTypes.STRING())
+      .field("f2", DataTypes.INT())
       .field("f3", DataTypes.TIMESTAMP(3))
+      .field("f4", DataTypes.TIMESTAMP(3))
+      .field("f5", DataTypes.INT())
       .build()
     val schema = SchemaValidator.deriveTableSinkSchema(properties)
     assertEquals(expectd, schema)