You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2018/10/30 10:26:10 UTC
[flink] branch master updated (f2b549f -> 9e5a427)
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from f2b549f [hotfix] [docs] Fix typo in ConfigGroups
new f3bce3c [FLINK-10687] [table] Introduce flink-table-common
new f0b244e [FLINK-10687] [table] Move TableException and ValidationException to flink-table-common
new 0ab12d7 [FLINK-10687] [table] Move TableSchema to flink-table-common
new 026f43a [FLINK-10687] [table] Move TypeStringUtils to flink-table-common
new 3bfe134 [FLINK-10687] [table] Move DescriptorProperties to flink-table-common
new 95b4086 [FLINK-10687] [table] Move format factories to flink-table-common
new 9e5a427 [FLINK-10687] [table] Move format descriptors and validators to flink-table-common
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/dev/table/sourceSinks.md | 20 +-
.../flink-connector-elasticsearch-base/pom.xml | 9 +
.../ElasticsearchUpsertTableSinkBase.java | 4 +-
.../ElasticsearchUpsertTableSinkFactoryBase.java | 10 +-
.../flink/table/descriptors/Elasticsearch.java | 10 +-
.../table/descriptors/ElasticsearchValidator.java | 28 +-
...lasticsearchUpsertTableSinkFactoryTestBase.java | 3 +-
.../flink-connector-kafka-base/pom.xml | 9 +
.../connectors/kafka/KafkaTableSinkBase.java | 4 +-
.../connectors/kafka/KafkaTableSourceBase.java | 12 +-
.../kafka/KafkaTableSourceSinkFactoryBase.java | 10 +-
.../org/apache/flink/table/descriptors/Kafka.java | 11 +-
.../flink/table/descriptors/KafkaValidator.java | 34 +-
.../kafka/KafkaJsonTableSourceFactoryTestBase.java | 8 +-
.../kafka/KafkaTableSourceBuilderTestBase.java | 22 +-
.../kafka/KafkaTableSourceSinkFactoryTestBase.java | 5 +-
.../org/apache/flink/orc/OrcTableSourceTest.java | 4 +-
flink-formats/flink-avro/pom.xml | 22 +-
.../flink/formats/avro/AvroRowFormatFactory.java | 29 +-
.../org/apache/flink/table/descriptors/Avro.java | 11 +-
.../flink/table/descriptors/AvroValidator.java | 2 +
.../formats/avro/AvroRowFormatFactoryTest.java | 14 +-
flink-formats/flink-json/pom.xml | 18 +-
.../flink/formats/json/JsonRowFormatFactory.java | 52 +-
.../org/apache/flink/table/descriptors/Json.java | 15 +-
.../flink/table/descriptors/JsonValidator.java | 8 +-
.../formats/json/JsonRowFormatFactoryTest.java | 6 +-
.../table/client/cli/CliChangelogResultView.java | 2 +-
.../flink/table/client/cli/CliResultView.java | 4 +-
.../flink/table/client/cli/CliTableResultView.java | 2 +-
.../table/client/config/entries/FunctionEntry.java | 4 +-
.../table/client/gateway/local/LocalExecutor.java | 22 +-
.../table/client/gateway/local/ResultStore.java | 2 +-
.../client/gateway/local/ExecutionContextTest.java | 10 +-
.../gateway/utils/TestTableSinkFactoryBase.java | 8 +-
.../gateway/utils/TestTableSourceFactoryBase.java | 4 +-
flink-libraries/flink-table-common/pom.xml | 58 +
.../apache/flink/table/api/TableException.java} | 26 +-
.../org/apache/flink/table/api/TableSchema.java | 275 ++++
.../flink/table/api/ValidationException.java} | 25 +-
.../table/descriptors/ConnectorDescriptor.java | 74 +
.../descriptors/ConnectorDescriptorValidator.java | 51 +
.../flink/table/descriptors/Descriptor.java} | 38 +-
.../flink/table/descriptors/DescriptorBase.java} | 21 +-
.../table/descriptors/DescriptorProperties.java | 1342 ++++++++++++++++
.../table/descriptors/DescriptorValidator.java} | 22 +-
.../flink/table/descriptors/FormatDescriptor.java | 60 +
.../descriptors/FormatDescriptorValidator.java | 61 +
.../factories/DeserializationSchemaFactory.java} | 34 +-
.../factories/SerializationSchemaFactory.java} | 34 +-
.../apache/flink/table/factories/TableFactory.java | 86 ++
.../flink/table/factories/TableFormatFactory.java | 68 +
.../table/factories/TableFormatFactoryBase.java | 165 ++
.../apache/flink/table/utils/EncodingUtils.java | 215 +++
.../apache/flink/table/utils/TypeStringUtils.java | 495 ++++++
.../table/descriptors/DescriptorTestBase.java | 87 ++
.../factories/TableFormatFactoryBaseTest.java | 85 ++
.../flink/table/utils/TypeStringUtilsTest.java | 179 +++
flink-libraries/flink-table/pom.xml | 14 +
.../flink/table/api/BatchTableEnvironment.scala | 6 +-
.../flink/table/api/StreamTableEnvironment.scala | 10 +-
.../apache/flink/table/api/TableEnvironment.scala | 21 +-
.../org/apache/flink/table/api/TableSchema.scala | 218 ---
.../org/apache/flink/table/api/exceptions.scala | 43 +-
.../scala/org/apache/flink/table/api/table.scala | 16 +-
.../flink/table/calcite/FlinkPlannerImpl.scala | 2 +-
.../flink/table/calcite/FlinkTypeFactory.scala | 13 +-
.../flink/table/catalog/ExternalCatalogTable.scala | 34 +-
.../apache/flink/table/codegen/CodeGenerator.scala | 2 +-
.../flink/table/descriptors/ClassInstance.scala | 11 +-
.../table/descriptors/ClassInstanceValidator.scala | 2 +-
.../table/descriptors/ConnectTableDescriptor.scala | 19 +-
.../table/descriptors/ConnectorDescriptor.scala | 55 -
.../descriptors/ConnectorDescriptorValidator.scala | 53 -
.../org/apache/flink/table/descriptors/Csv.scala | 37 +-
.../flink/table/descriptors/CsvValidator.scala | 16 +-
.../flink/table/descriptors/Descriptor.scala | 41 -
.../table/descriptors/DescriptorProperties.scala | 1601 --------------------
.../flink/table/descriptors/FileSystem.scala | 17 +-
.../table/descriptors/FileSystemValidator.scala | 4 +-
.../flink/table/descriptors/FormatDescriptor.scala | 49 -
.../descriptors/FormatDescriptorValidator.scala | 63 -
.../table/descriptors/FunctionDescriptor.scala | 12 +-
.../descriptors/FunctionDescriptorValidator.scala | 4 +-
.../flink/table/descriptors/LiteralValue.scala | 13 +-
.../table/descriptors/LiteralValueValidator.scala | 26 +-
.../apache/flink/table/descriptors/Metadata.scala | 11 +-
.../table/descriptors/MetadataValidator.scala | 8 +-
.../table/descriptors/RegistrableDescriptor.scala | 2 +-
.../apache/flink/table/descriptors/Rowtime.scala | 21 +-
.../flink/table/descriptors/RowtimeValidator.scala | 35 +-
.../apache/flink/table/descriptors/Schema.scala | 25 +-
.../flink/table/descriptors/SchemaValidator.scala | 46 +-
.../table/descriptors/SchematicDescriptor.scala | 2 +-
.../flink/table/descriptors/Statistics.scala | 13 +-
.../table/descriptors/StatisticsValidator.scala | 20 +-
.../table/descriptors/StreamTableDescriptor.scala | 13 +-
.../StreamTableDescriptorValidator.scala | 2 +-
.../flink/table/descriptors/TableDescriptor.scala | 4 +-
.../flink/table/expressions/ExpressionUtils.scala | 2 +-
.../org/apache/flink/table/expressions/call.scala | 2 +-
.../flink/table/expressions/fieldExpression.scala | 2 +-
.../flink/table/factories/TableFactory.scala | 85 --
.../table/factories/TableFactoryService.scala | 4 +-
.../flink/table/factories/TableFactoryUtil.scala | 6 +-
.../flink/table/factories/TableFormatFactory.scala | 65 -
.../flink/table/functions/FunctionService.scala | 8 +-
.../table/functions/TemporalTableFunction.scala | 4 +-
.../functions/utils/UserDefinedFunctionUtils.scala | 9 +-
.../flink/table/plan/nodes/CommonCorrelate.scala | 2 +-
.../flink/table/plan/nodes/FlinkRelNode.scala | 2 +-
.../table/plan/nodes/dataset/DataSetJoin.scala | 10 +-
.../table/plan/nodes/dataset/DataSetSort.scala | 2 +-
.../DataStreamGroupWindowAggregate.scala | 2 +-
.../plan/nodes/datastream/DataStreamJoin.scala | 2 +-
.../plan/nodes/datastream/DataStreamMatch.scala | 12 +-
.../logical/FlinkLogicalTableSourceScan.scala | 2 +-
.../rules/common/LogicalWindowAggregateRule.scala | 6 +-
.../plan/rules/common/WindowPropertiesRule.scala | 5 +-
.../rules/dataSet/DataSetWindowAggregateRule.scala | 4 +-
.../datastream/DataStreamGroupAggregateRule.scala | 2 +-
.../DataStreamGroupWindowAggregateRule.scala | 2 +-
.../DataStreamLogicalWindowAggregateRule.scala | 6 +-
.../plan/rules/logical/LogicalUnnestRule.scala | 4 +-
.../table/runtime/aggregate/AggregateUtil.scala | 13 +-
.../table/sinks/CsvTableSinkFactoryBase.scala | 12 +-
.../flink/table/sources/CsvTableSource.scala | 2 +-
.../table/sources/CsvTableSourceFactoryBase.scala | 12 +-
.../apache/flink/table/sources/TableSource.scala | 2 +-
.../flink/table/sources/TableSourceUtil.scala | 53 +-
.../table/sources/tsextractors/ExistingField.scala | 2 +-
.../flink/table/typeutils/TypeStringUtils.scala | 299 ----
.../flink/table/util/JavaScalaConversionUtil.scala | 61 +
.../flink/table/validate/FunctionCatalog.scala | 14 +-
.../apache/flink/table/api/TableSchemaTest.scala | 28 +-
.../sql/validation/InsertIntoValidationTest.scala | 2 +-
.../sql/validation/InsertIntoValidationTest.scala | 2 +-
.../api/stream/table/TemporalTableJoinTest.scala | 4 +-
.../api/validation/TableSchemaValidationTest.scala | 8 +-
.../catalog/InMemoryExternalCatalogTest.scala | 10 +-
.../descriptors/DescriptorPropertiesTest.scala | 20 +-
.../table/descriptors/DescriptorTestBase.scala | 128 --
.../flink/table/descriptors/RowtimeTest.scala | 2 +-
.../table/descriptors/SchemaValidatorTest.scala | 27 +-
.../table/descriptors/TableDescriptorTest.scala | 5 +-
.../table/descriptors/TestTableDescriptor.scala | 69 +
.../table/factories/utils/TestTableFormat.scala | 8 +-
.../factories/utils/TestTableFormatFactory.scala | 32 +-
.../runtime/batch/sql/TableEnvironmentITCase.scala | 2 +-
.../batch/table/TableEnvironmentITCase.scala | 2 +-
.../flink/table/runtime/stream/sql/SqlITCase.scala | 14 +-
.../table/typeutils/TypeStringUtilsTest.scala | 124 --
.../flink/table/utils/InMemoryTableFactory.scala | 4 +-
.../flink/table/utils/testTableSources.scala | 2 +-
flink-libraries/pom.xml | 1 +
155 files changed, 4232 insertions(+), 3525 deletions(-)
create mode 100644 flink-libraries/flink-table-common/pom.xml
copy flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala => flink-table-common/src/main/java/org/apache/flink/table/api/TableException.java} (60%)
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableSchema.java
copy flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala => flink-table-common/src/main/java/org/apache/flink/table/api/ValidationException.java} (64%)
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java
copy flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystemValidator.scala => flink-table-common/src/main/java/org/apache/flink/table/descriptors/Descriptor.java} (52%)
copy flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala => flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorBase.java} (70%)
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java
rename flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala => flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorValidator.java} (67%)
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptor.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptorValidator.java
rename flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/factories/DeserializationSchemaFactory.scala => flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java} (54%)
rename flink-libraries/{flink-table/src/main/scala/org/apache/flink/table/factories/SerializationSchemaFactory.scala => flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java} (54%)
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
create mode 100644 flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/TypeStringUtils.java
create mode 100644 flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/descriptors/DescriptorTestBase.java
create mode 100644 flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/factories/TableFormatFactoryBaseTest.java
create mode 100644 flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/utils/TypeStringUtilsTest.java
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Descriptor.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactory.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFormatFactory.scala
delete mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeStringUtils.scala
create mode 100644 flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
delete mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
create mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TestTableDescriptor.scala
delete mode 100644 flink-libraries/flink-table/src/test/scala/org/apache/flink/table/typeutils/TypeStringUtilsTest.scala
[flink] 03/07: [FLINK-10687] [table] Move TableSchema to
flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 0ab12d716171227f2a3956bf805e065434d6fbd9
Author: Timo Walther <tw...@apache.org>
AuthorDate: Fri Oct 26 13:34:19 2018 +0200
[FLINK-10687] [table] Move TableSchema to flink-table-common
---
.../ElasticsearchUpsertTableSinkBase.java | 4 +-
.../connectors/kafka/KafkaTableSinkBase.java | 4 +-
.../connectors/kafka/KafkaTableSourceBase.java | 12 +-
.../kafka/KafkaTableSourceSinkFactoryBase.java | 2 +-
.../kafka/KafkaTableSourceBuilderTestBase.java | 22 +-
.../org/apache/flink/orc/OrcTableSourceTest.java | 4 +-
.../table/client/cli/CliChangelogResultView.java | 2 +-
.../flink/table/client/cli/CliResultView.java | 4 +-
.../flink/table/client/cli/CliTableResultView.java | 2 +-
.../table/client/gateway/local/LocalExecutor.java | 22 +-
.../table/client/gateway/local/ResultStore.java | 2 +-
.../client/gateway/local/ExecutionContextTest.java | 10 +-
.../gateway/utils/TestTableSinkFactoryBase.java | 6 +-
.../gateway/utils/TestTableSourceFactoryBase.java | 2 +-
.../org/apache/flink/table/api/TableSchema.java | 275 +++++++++++++++++++++
.../apache/flink/table/api/TableEnvironment.scala | 4 +-
.../org/apache/flink/table/api/TableSchema.scala | 218 ----------------
.../table/descriptors/DescriptorProperties.scala | 2 +-
.../flink/table/descriptors/SchemaValidator.scala | 6 +-
.../table/functions/TemporalTableFunction.scala | 4 +-
.../logical/FlinkLogicalTableSourceScan.scala | 2 +-
.../table/sinks/CsvTableSinkFactoryBase.scala | 2 +-
.../table/sources/CsvTableSourceFactoryBase.scala | 2 +-
.../apache/flink/table/sources/TableSource.scala | 2 +-
.../flink/table/sources/TableSourceUtil.scala | 26 +-
.../apache/flink/table/api/TableSchemaTest.scala | 28 +--
.../sql/validation/InsertIntoValidationTest.scala | 2 +-
.../sql/validation/InsertIntoValidationTest.scala | 2 +-
.../api/stream/table/TemporalTableJoinTest.scala | 4 +-
.../api/validation/TableSchemaValidationTest.scala | 8 +-
.../runtime/batch/sql/TableEnvironmentITCase.scala | 2 +-
.../batch/table/TableEnvironmentITCase.scala | 2 +-
.../flink/table/utils/InMemoryTableFactory.scala | 4 +-
.../flink/table/utils/testTableSources.scala | 2 +-
34 files changed, 384 insertions(+), 311 deletions(-)
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkBase.java
index 9bee8c3..8e0279f 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkBase.java
@@ -195,12 +195,12 @@ public abstract class ElasticsearchUpsertTableSinkBase implements UpsertStreamTa
@Override
public String[] getFieldNames() {
- return schema.getColumnNames();
+ return schema.getFieldNames();
}
@Override
public TypeInformation<?>[] getFieldTypes() {
- return schema.getTypes();
+ return schema.getFieldTypes();
}
@Override
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkBase.java
index acd10cc..4755cff 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSinkBase.java
@@ -155,12 +155,12 @@ public abstract class KafkaTableSinkBase implements AppendStreamTableSink<Row> {
}
public String[] getFieldNames() {
- return schema.map(TableSchema::getColumnNames).orElse(fieldNames);
+ return schema.map(TableSchema::getFieldNames).orElse(fieldNames);
}
@Override
public TypeInformation<?>[] getFieldTypes() {
- return schema.map(TableSchema::getTypes).orElse(fieldTypes);
+ return schema.map(TableSchema::getFieldTypes).orElse(fieldTypes);
}
@Override
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java
index 04aa8c1..7078fc9 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBase.java
@@ -48,8 +48,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
-import scala.Option;
-
/**
* A version-agnostic Kafka {@link StreamTableSource}.
*
@@ -197,7 +195,7 @@ public abstract class KafkaTableSourceBase implements
@Override
public String explainSource() {
- return TableConnectorUtil.generateRuntimeName(this.getClass(), schema.getColumnNames());
+ return TableConnectorUtil.generateRuntimeName(this.getClass(), schema.getFieldNames());
}
/**
@@ -295,8 +293,8 @@ public abstract class KafkaTableSourceBase implements
private Optional<String> validateProctimeAttribute(Optional<String> proctimeAttribute) {
return proctimeAttribute.map((attribute) -> {
// validate that field exists and is of correct type
- Option<TypeInformation<?>> tpe = schema.getType(attribute);
- if (tpe.isEmpty()) {
+ Optional<TypeInformation<?>> tpe = schema.getFieldType(attribute);
+ if (!tpe.isPresent()) {
throw new ValidationException("Processing time attribute '" + attribute + "' is not present in TableSchema.");
} else if (tpe.get() != Types.SQL_TIMESTAMP()) {
throw new ValidationException("Processing time attribute '" + attribute + "' is not of type SQL_TIMESTAMP.");
@@ -315,8 +313,8 @@ public abstract class KafkaTableSourceBase implements
// validate that all declared fields exist and are of correct type
for (RowtimeAttributeDescriptor desc : rowtimeAttributeDescriptors) {
String rowtimeAttribute = desc.getAttributeName();
- Option<TypeInformation<?>> tpe = schema.getType(rowtimeAttribute);
- if (tpe.isEmpty()) {
+ Optional<TypeInformation<?>> tpe = schema.getFieldType(rowtimeAttribute);
+ if (!tpe.isPresent()) {
throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not present in TableSchema.");
} else if (tpe.get() != Types.SQL_TIMESTAMP()) {
throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not of type SQL_TIMESTAMP.");
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
index 431f946..b4aea0f 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
@@ -346,7 +346,7 @@ public abstract class KafkaTableSourceSinkFactoryBase implements
final Map<String, String> fieldMapping = SchemaValidator.deriveFieldMapping(
descriptorProperties,
Optional.of(schema.toRowType())); // until FLINK-9870 is fixed we assume that the table schema is the output type
- return fieldMapping.size() != schema.getColumnNames().length ||
+ return fieldMapping.size() != schema.getFieldNames().length ||
!fieldMapping.entrySet().stream().allMatch(mapping -> mapping.getKey().equals(mapping.getValue()));
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBuilderTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBuilderTestBase.java
index 16dc714..f778ba6 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBuilderTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceBuilderTestBase.java
@@ -97,18 +97,18 @@ public abstract class KafkaTableSourceBuilderTestBase {
// check table schema
TableSchema schema = source.getTableSchema();
assertNotNull(schema);
- assertEquals(5, schema.getColumnNames().length);
+ assertEquals(5, schema.getFieldNames().length);
// check table fields
- assertEquals("field1", schema.getColumnNames()[0]);
- assertEquals("field2", schema.getColumnNames()[1]);
- assertEquals("time1", schema.getColumnNames()[2]);
- assertEquals("time2", schema.getColumnNames()[3]);
- assertEquals("field3", schema.getColumnNames()[4]);
- assertEquals(Types.LONG(), schema.getTypes()[0]);
- assertEquals(Types.STRING(), schema.getTypes()[1]);
- assertEquals(Types.SQL_TIMESTAMP(), schema.getTypes()[2]);
- assertEquals(Types.SQL_TIMESTAMP(), schema.getTypes()[3]);
- assertEquals(Types.DOUBLE(), schema.getTypes()[4]);
+ assertEquals("field1", schema.getFieldNames()[0]);
+ assertEquals("field2", schema.getFieldNames()[1]);
+ assertEquals("time1", schema.getFieldNames()[2]);
+ assertEquals("time2", schema.getFieldNames()[3]);
+ assertEquals("field3", schema.getFieldNames()[4]);
+ assertEquals(Types.LONG(), schema.getFieldTypes()[0]);
+ assertEquals(Types.STRING(), schema.getFieldTypes()[1]);
+ assertEquals(Types.SQL_TIMESTAMP(), schema.getFieldTypes()[2]);
+ assertEquals(Types.SQL_TIMESTAMP(), schema.getFieldTypes()[3]);
+ assertEquals(Types.DOUBLE(), schema.getFieldTypes()[4]);
}
@Test
diff --git a/flink-connectors/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceTest.java b/flink-connectors/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceTest.java
index cd49815..364c5d5 100644
--- a/flink-connectors/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceTest.java
+++ b/flink-connectors/flink-orc/src/test/java/org/apache/flink/orc/OrcTableSourceTest.java
@@ -128,8 +128,8 @@ public class OrcTableSourceTest {
TableSchema schema = orc.getTableSchema();
assertNotNull(schema);
- assertArrayEquals(getNestedFieldNames(), schema.getColumnNames());
- assertArrayEquals(getNestedFieldTypes(), schema.getTypes());
+ assertArrayEquals(getNestedFieldNames(), schema.getFieldNames());
+ assertArrayEquals(getNestedFieldTypes(), schema.getFieldTypes());
}
@Test
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java
index d70af80..5388a3e 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliChangelogResultView.java
@@ -249,7 +249,7 @@ public class CliChangelogResultView extends CliResultView<CliChangelogResultView
schemaHeader.append("+/-");
schemaHeader.style(AttributedStyle.DEFAULT);
- Arrays.stream(resultDescriptor.getResultSchema().getColumnNames()).forEach(s -> {
+ Arrays.stream(resultDescriptor.getResultSchema().getFieldNames()).forEach(s -> {
schemaHeader.append(' ');
schemaHeader.style(AttributedStyle.DEFAULT.underline());
normalizeColumn(schemaHeader, s, MAX_COLUMN_WIDTH);
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java
index 78f3a56..1fdd005 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliResultView.java
@@ -144,8 +144,8 @@ public abstract class CliResultView<O extends Enum<O>> extends CliView<O, Void>
}
final CliRowView view = new CliRowView(
client,
- resultDescriptor.getResultSchema().getColumnNames(),
- CliUtils.typesToString(resultDescriptor.getResultSchema().getTypes()),
+ resultDescriptor.getResultSchema().getFieldNames(),
+ CliUtils.typesToString(resultDescriptor.getResultSchema().getFieldTypes()),
getRow(results.get(selectedRow)));
view.open(); // enter view
}
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java
index 2c630d3..9c218f6 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliTableResultView.java
@@ -231,7 +231,7 @@ public class CliTableResultView extends CliResultView<CliTableResultView.ResultT
protected List<AttributedString> computeMainHeaderLines() {
final AttributedStringBuilder schemaHeader = new AttributedStringBuilder();
- Arrays.stream(resultDescriptor.getResultSchema().getColumnNames()).forEach(s -> {
+ Arrays.stream(resultDescriptor.getResultSchema().getFieldNames()).forEach(s -> {
schemaHeader.append(' ');
schemaHeader.style(AttributedStyle.DEFAULT.underline());
normalizeColumn(schemaHeader, s, MAX_COLUMN_WIDTH);
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
index e8dd021..36fd4ee 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
@@ -19,6 +19,8 @@
package org.apache.flink.table.client.gateway.local;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.client.cli.CliFrontend;
import org.apache.flink.client.cli.CliFrontendParser;
@@ -35,6 +37,7 @@ import org.apache.flink.table.api.QueryConfig;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.calcite.FlinkTypeFactory;
import org.apache.flink.table.client.SqlClientException;
import org.apache.flink.table.client.config.Environment;
import org.apache.flink.table.client.gateway.Executor;
@@ -404,7 +407,7 @@ public class LocalExecutor implements Executor {
// initialize result
final DynamicResult<C> result = resultStore.createResult(
context.getMergedEnvironment(),
- table.getSchema().withoutTimeAttributes(),
+ removeTimeAttributes(table.getSchema()),
envInst.getExecutionConfig());
// create job graph with dependencies
@@ -438,7 +441,7 @@ public class LocalExecutor implements Executor {
return new ResultDescriptor(
resultId,
- table.getSchema().withoutTimeAttributes(),
+ removeTimeAttributes(table.getSchema()),
result.isMaterialized());
}
@@ -540,4 +543,19 @@ public class LocalExecutor implements Executor {
CliFrontendParser.getRunCommandOptions(),
customOptions);
}
+
+ private static TableSchema removeTimeAttributes(TableSchema schema) {
+ final TableSchema.Builder builder = TableSchema.builder();
+ for (int i = 0; i < schema.getFieldCount(); i++) {
+ final TypeInformation<?> type = schema.getFieldTypes()[i];
+ final TypeInformation<?> convertedType;
+ if (FlinkTypeFactory.isTimeIndicatorType(type)) {
+ convertedType = Types.SQL_TIMESTAMP;
+ } else {
+ convertedType = type;
+ }
+ builder.field(schema.getFieldNames()[i], convertedType);
+ }
+ return builder.build();
+ }
}
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java
index a25c2bf..7a24325 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ResultStore.java
@@ -61,7 +61,7 @@ public class ResultStore {
*/
public <T> DynamicResult<T> createResult(Environment env, TableSchema schema, ExecutionConfig config) {
- final TypeInformation<Row> outputType = Types.ROW_NAMED(schema.getColumnNames(), schema.getTypes());
+ final TypeInformation<Row> outputType = Types.ROW_NAMED(schema.getFieldNames(), schema.getFieldTypes());
if (env.getExecution().isStreamingExecution()) {
// determine gateway address (and port if possible)
diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java
index c51e930..22aadc6 100644
--- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java
+++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/ExecutionContextTest.java
@@ -97,19 +97,19 @@ public class ExecutionContextTest {
assertArrayEquals(
new String[]{"IntegerField1", "StringField1"},
- sources.get("TableNumber1").getTableSchema().getColumnNames());
+ sources.get("TableNumber1").getTableSchema().getFieldNames());
assertArrayEquals(
new TypeInformation[]{Types.INT(), Types.STRING()},
- sources.get("TableNumber1").getTableSchema().getTypes());
+ sources.get("TableNumber1").getTableSchema().getFieldTypes());
assertArrayEquals(
new String[]{"IntegerField2", "StringField2"},
- sources.get("TableNumber2").getTableSchema().getColumnNames());
+ sources.get("TableNumber2").getTableSchema().getFieldNames());
assertArrayEquals(
new TypeInformation[]{Types.INT(), Types.STRING()},
- sources.get("TableNumber2").getTableSchema().getTypes());
+ sources.get("TableNumber2").getTableSchema().getFieldTypes());
assertArrayEquals(
new String[]{"BooleanField", "StringField"},
@@ -146,7 +146,7 @@ public class ExecutionContextTest {
assertArrayEquals(
new String[]{"integerField", "stringField", "rowtimeField", "integerField0", "stringField0", "rowtimeField0"},
- tableEnv.scan("TemporalTableUsage").getSchema().getColumnNames());
+ tableEnv.scan("TemporalTableUsage").getSchema().getFieldNames());
}
private <T> ExecutionContext<T> createExecutionContext(String file, Map<String, String> replaceVars) throws Exception {
diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
index 21ce4db..1de2407 100644
--- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
+++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
@@ -108,17 +108,17 @@ public abstract class TestTableSinkFactoryBase implements StreamTableSinkFactory
@Override
public TypeInformation<Row> getOutputType() {
- return Types.ROW(schema.getColumnNames(), schema.getTypes());
+ return Types.ROW(schema.getFieldNames(), schema.getFieldTypes());
}
@Override
public String[] getFieldNames() {
- return schema.getColumnNames();
+ return schema.getFieldNames();
}
@Override
public TypeInformation<?>[] getFieldTypes() {
- return schema.getTypes();
+ return schema.getFieldTypes();
}
@Override
diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
index 400ae60..f6ad16d 100644
--- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
+++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
@@ -124,7 +124,7 @@ public abstract class TestTableSourceFactoryBase implements StreamTableSourceFac
@Override
public TypeInformation<Row> getReturnType() {
- return Types.ROW(schema.getColumnNames(), schema.getTypes());
+ return Types.ROW(schema.getFieldNames(), schema.getFieldTypes());
}
@Override
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableSchema.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableSchema.java
new file mode 100644
index 0000000..2e95223
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableSchema.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.Preconditions;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * A table schema that represents a table's structure with field names and types.
+ */
+@PublicEvolving
+public class TableSchema {
+
+ private static final String ATOMIC_TYPE_FIELD_NAME = "f0";
+
+ private final String[] fieldNames;
+
+ private final TypeInformation<?>[] fieldTypes;
+
+ private final Map<String, Integer> fieldNameToIndex;
+
+ public TableSchema(String[] fieldNames, TypeInformation<?>[] fieldTypes) {
+ this.fieldNames = Preconditions.checkNotNull(fieldNames);
+ this.fieldTypes = Preconditions.checkNotNull(fieldTypes);
+
+ if (fieldNames.length != fieldTypes.length) {
+ throw new TableException(
+ "Number of field names and field types must be equal.\n" +
+ "Number of names is " + fieldNames.length + ", number of types is " + fieldTypes.length + ".\n" +
+ "List of field names: " + Arrays.toString(fieldNames) + "\n" +
+ "List of field types: " + Arrays.toString(fieldTypes));
+ }
+
+ // validate and create name to index mapping
+ fieldNameToIndex = new HashMap<>();
+ final Set<String> duplicateNames = new HashSet<>();
+ final Set<String> uniqueNames = new HashSet<>();
+ for (int i = 0; i < fieldNames.length; i++) {
+ // check for null
+ Preconditions.checkNotNull(fieldTypes[i]);
+ final String fieldName = Preconditions.checkNotNull(fieldNames[i]);
+
+ // collect indices
+ fieldNameToIndex.put(fieldName, i);
+
+ // check uniqueness of field names
+ if (uniqueNames.contains(fieldName)) {
+ duplicateNames.add(fieldName);
+ } else {
+ uniqueNames.add(fieldName);
+ }
+ }
+ if (!duplicateNames.isEmpty()) {
+ throw new TableException(
+ "Field names must be unique.\n" +
+ "List of duplicate fields: " + duplicateNames.toString() + "\n" +
+ "List of all fields: " + Arrays.toString(fieldNames));
+ }
+ }
+
+ /**
+ * Returns a deep copy of the table schema.
+ */
+ public TableSchema copy() {
+ return new TableSchema(fieldNames.clone(), fieldTypes.clone());
+ }
+
+ /**
+ * Returns all field type information as an array.
+ */
+ public TypeInformation<?>[] getFieldTypes() {
+ return fieldTypes;
+ }
+
+ /**
+ * Returns the specified type information for the given field index.
+ *
+ * @param fieldIndex the index of the field
+ */
+ public Optional<TypeInformation<?>> getFieldType(int fieldIndex) {
+ if (fieldIndex < 0 || fieldIndex >= fieldTypes.length) {
+ return Optional.empty();
+ }
+ return Optional.of(fieldTypes[fieldIndex]);
+ }
+
+ /**
+ * Returns the specified type information for the given field name.
+ *
+ * @param fieldName the name of the field
+ */
+ public Optional<TypeInformation<?>> getFieldType(String fieldName) {
+ if (fieldNameToIndex.containsKey(fieldName)) {
+ return Optional.of(fieldTypes[fieldNameToIndex.get(fieldName)]);
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * Returns the number of fields.
+ */
+ public int getFieldCount() {
+ return fieldNames.length;
+ }
+
+ /**
+ * Returns all field names as an array.
+ */
+ public String[] getFieldNames() {
+ return fieldNames;
+ }
+
+ /**
+ * Returns the specified name for the given field index.
+ *
+ * @param fieldIndex the index of the field
+ */
+ public Optional<String> getFieldName(int fieldIndex) {
+ if (fieldIndex < 0 || fieldIndex >= fieldNames.length) {
+ return Optional.empty();
+ }
+ return Optional.of(fieldNames[fieldIndex]);
+ }
+
+ /**
+ * @deprecated Use {@link TableSchema#getFieldTypes()} instead. Can be dropped after 1.7.
+ */
+ @Deprecated
+ public TypeInformation<?>[] getTypes() {
+ return getFieldTypes();
+ }
+
+ /**
+ * @deprecated Use {@link TableSchema#getFieldNames()} instead. Can be dropped after 1.7.
+ */
+ @Deprecated
+ public String[] getColumnNames() {
+ return getFieldNames();
+ }
+
+ /**
+ * Converts a table schema into a (nested) type information describing a {@link Row}.
+ */
+ public TypeInformation<Row> toRowType() {
+ return Types.ROW_NAMED(fieldNames, fieldTypes);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("root\n");
+ for (int i = 0; i < fieldNames.length; i++) {
+ sb.append(" |-- ").append(fieldNames[i]).append(": ").append(fieldTypes[i]).append('\n');
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TableSchema schema = (TableSchema) o;
+ return Arrays.equals(fieldNames, schema.fieldNames) &&
+ Arrays.equals(fieldTypes, schema.fieldTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode(fieldNames);
+ result = 31 * result + Arrays.hashCode(fieldTypes);
+ return result;
+ }
+
+ /**
+ * Creates a table schema from a {@link TypeInformation} instance. If the type information is
+ * a {@link CompositeType}, the field names and types for the composite type are used to
+ * construct the {@link TableSchema} instance. Otherwise, a table schema with a single field
+ * is created. The field name is "f0" and the field type the provided type.
+ *
+ * @param typeInfo The {@link TypeInformation} from which the table schema is generated.
+ * @return The table schema that was generated from the given {@link TypeInformation}.
+ */
+ public static TableSchema fromTypeInfo(TypeInformation<?> typeInfo) {
+ if (typeInfo instanceof CompositeType<?>) {
+ final CompositeType<?> compositeType = (CompositeType<?>) typeInfo;
+ // get field names and types from composite type
+ final String[] fieldNames = compositeType.getFieldNames();
+ final TypeInformation<?>[] fieldTypes = new TypeInformation[fieldNames.length];
+ for (int i = 0; i < fieldTypes.length; i++) {
+ fieldTypes[i] = compositeType.getTypeAt(i);
+ }
+ return new TableSchema(fieldNames, fieldTypes);
+ } else {
+ // create table schema with a single field named "f0" of the given type.
+ return new TableSchema(
+ new String[]{ATOMIC_TYPE_FIELD_NAME},
+ new TypeInformation<?>[]{typeInfo});
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Builder for creating a {@link TableSchema}.
+ */
+ public static class Builder {
+
+ private List<String> fieldNames;
+
+ private List<TypeInformation<?>> fieldTypes;
+
+ public Builder() {
+ fieldNames = new ArrayList<>();
+ fieldTypes = new ArrayList<>();
+ }
+
+ /**
+ * Add a field with name and type. The call order of this method determines the order
+ * of fields in the schema.
+ */
+ public Builder field(String name, TypeInformation<?> type) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(type);
+ fieldNames.add(name);
+ fieldTypes.add(type);
+ return this;
+ }
+
+ /**
+ * Returns a {@link TableSchema} instance.
+ */
+ public TableSchema build() {
+ return new TableSchema(
+ fieldNames.toArray(new String[0]),
+ fieldTypes.toArray(new TypeInformation<?>[0]));
+ }
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 3cc97d3..58831d1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -828,13 +828,13 @@ abstract class TableEnvironment(val config: TableConfig) {
case Some(s: TableSourceSinkTable[_, _]) if s.tableSinkTable.isDefined =>
val tableSink = s.tableSinkTable.get.tableSink
// validate schema of source table and table sink
- val srcFieldTypes = table.getSchema.getTypes
+ val srcFieldTypes = table.getSchema.getFieldTypes
val sinkFieldTypes = tableSink.getFieldTypes
if (srcFieldTypes.length != sinkFieldTypes.length ||
srcFieldTypes.zip(sinkFieldTypes).exists { case (srcF, snkF) => srcF != snkF }) {
- val srcFieldNames = table.getSchema.getColumnNames
+ val srcFieldNames = table.getSchema.getFieldNames
val sinkFieldNames = tableSink.getFieldNames
// format table and table sink schema strings
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
deleted file mode 100644
index bcee5ac..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.api
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.typeutils.CompositeType
-
-import _root_.scala.collection.mutable.ArrayBuffer
-import _root_.java.util.Objects
-
-import org.apache.flink.table.calcite.FlinkTypeFactory
-import org.apache.flink.types.Row
-
-/**
- * A TableSchema represents a Table's structure.
- */
-class TableSchema(
- private val columnNames: Array[String],
- private val columnTypes: Array[TypeInformation[_]]) {
-
- private val columnNameToIndex: Map[String, Int] = columnNames.zipWithIndex.toMap
-
- if (columnNames.length != columnTypes.length) {
- throw new TableException(
- s"Number of field names and field types must be equal.\n" +
- s"Number of names is ${columnNames.length}, number of types is ${columnTypes.length}.\n" +
- s"List of field names: ${columnNames.mkString("[", ", ", "]")}.\n" +
- s"List of field types: ${columnTypes.mkString("[", ", ", "]")}.")
- }
-
- // check uniqueness of field names
- if (columnNames.toSet.size != columnTypes.length) {
- val duplicateFields = columnNames
- // count occurrences of field names
- .groupBy(identity).mapValues(_.length)
- // filter for occurrences > 1 and map to field name
- .filter(g => g._2 > 1).keys
-
- throw new TableException(
- s"Field names must be unique.\n" +
- s"List of duplicate fields: ${duplicateFields.mkString("[", ", ", "]")}.\n" +
- s"List of all fields: ${columnNames.mkString("[", ", ", "]")}.")
- }
-
- /**
- * Returns a deep copy of the TableSchema.
- */
- def copy: TableSchema = {
- new TableSchema(columnNames.clone(), columnTypes.clone())
- }
-
- /**
- * Returns all type information as an array.
- */
- def getTypes: Array[TypeInformation[_]] = columnTypes
-
- /**
- * Returns the specified type information for the given column index.
- *
- * @param columnIndex the index of the field
- */
- def getType(columnIndex: Int): Option[TypeInformation[_]] = {
- if (columnIndex < 0 || columnIndex >= columnNames.length) {
- None
- } else {
- Some(columnTypes(columnIndex))
- }
- }
-
- /**
- * Returns the specified type information for the given column name.
- *
- * @param columnName the name of the field
- */
- def getType(columnName: String): Option[TypeInformation[_]] = {
- if (columnNameToIndex.contains(columnName)) {
- Some(columnTypes(columnNameToIndex(columnName)))
- } else {
- None
- }
- }
-
- /**
- * Returns the number of columns.
- */
- def getColumnCount: Int = columnNames.length
-
- /**
- * Returns all column names as an array.
- */
- def getColumnNames: Array[String] = columnNames
-
- /**
- * Returns the specified column name for the given column index.
- *
- * @param columnIndex the index of the field
- */
- def getColumnName(columnIndex: Int): Option[String] = {
- if (columnIndex < 0 || columnIndex >= columnNames.length) {
- None
- } else {
- Some(columnNames(columnIndex))
- }
- }
-
- /**
- * Converts a table schema into a schema that represents the result that would be written
- * into a table sink or operator outside of the Table & SQL API. Time attributes are replaced
- * by proper TIMESTAMP data types.
- *
- * @return a table schema with no time attributes
- */
- def withoutTimeAttributes: TableSchema = {
- val converted = columnTypes.map { t =>
- if (FlinkTypeFactory.isTimeIndicatorType(t)) {
- Types.SQL_TIMESTAMP
- } else {
- t
- }
- }
- new TableSchema(columnNames, converted)
- }
-
- /**
- * Converts a table schema into a (nested) type information describing a [[Row]].
- *
- * @return type information where columns are fields of a row
- */
- def toRowType: TypeInformation[Row] = {
- Types.ROW(getColumnNames, getTypes)
- }
-
- override def toString: String = {
- val builder = new StringBuilder
- builder.append("root\n")
- columnNames.zip(columnTypes).foreach{ case (name, typeInfo) =>
- builder.append(s" |-- $name: $typeInfo\n")
- }
- builder.toString()
- }
-
- override def equals(other: Any): Boolean = {
- other match {
- case that: TableSchema if that.canEqual(this) =>
- this.columnNames.deep == that.columnNames.deep &&
- this.columnTypes.deep == that.columnTypes.deep
- case _ => false
- }
- }
-
- def canEqual(other: Any): Boolean = other.isInstanceOf[TableSchema]
-
- override def hashCode(): Int = {
- Objects.hash(columnNames, columnTypes)
- }
-}
-
-object TableSchema {
-
- /**
- * Creates a [[TableSchema]] from a [[TypeInformation]].
- * If the [[TypeInformation]] is a [[CompositeType]], the fieldnames and types for the composite
- * type are used to construct the [[TableSchema]].
- * Otherwise, a [[TableSchema]] with a single field is created. The field name is "f0" and the
- * field type the provided type.
- *
- * @param typeInfo The [[TypeInformation]] from which the [[TableSchema]] is generated.
- * @return The [[TableSchema]] that was generated from the given [[TypeInformation]].
- */
- def fromTypeInfo(typeInfo: TypeInformation[_]): TableSchema = {
- typeInfo match {
- case c: CompositeType[_] =>
- // get field names and types from composite type
- val fieldNames = c.getFieldNames
- val fieldTypes = fieldNames.map(c.getTypeAt).asInstanceOf[Array[TypeInformation[_]]]
- new TableSchema(fieldNames, fieldTypes)
- case t: TypeInformation[_] =>
- // create table schema with a single field named "f0" of the given type.
- new TableSchema(Array("f0"), Array(t))
- }
- }
-
- def builder(): TableSchemaBuilder = {
- new TableSchemaBuilder
- }
-
-}
-
-class TableSchemaBuilder {
-
- private val fieldNames: ArrayBuffer[String] = new ArrayBuffer[String]()
- private val fieldTypes: ArrayBuffer[TypeInformation[_]] = new ArrayBuffer[TypeInformation[_]]()
-
- def field(name: String, tpe: TypeInformation[_]): TableSchemaBuilder = {
- fieldNames.append(name)
- fieldTypes.append(tpe)
- this
- }
-
- def build(): TableSchema = {
- new TableSchema(fieldNames.toArray, fieldTypes.toArray)
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
index 21ff13c..e64410e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
@@ -1507,7 +1507,7 @@ object DescriptorProperties {
}
def normalizeTableSchema(schema: TableSchema): Seq[(String, String)] = {
- schema.getColumnNames.zip(schema.getTypes).map { case (n, t) =>
+ schema.getFieldNames.zip(schema.getFieldTypes).map { case (n, t) =>
(n, normalizeTypeInfo(t))
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
index f6cbb2b..f6279bb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
@@ -190,7 +190,7 @@ object SchemaValidator {
val schema = properties.getTableSchema(SCHEMA)
- schema.getColumnNames.zip(schema.getTypes).zipWithIndex.foreach { case ((n, t), i) =>
+ schema.getFieldNames.zip(schema.getFieldTypes).zipWithIndex.foreach { case ((n, t), i) =>
val isProctime = properties
.getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")
.orElse(false)
@@ -247,7 +247,7 @@ object SchemaValidator {
columnNames.foreach(name => mapping.put(name, name))
// add all schema fields first for implicit mappings
- schema.getColumnNames.foreach { name =>
+ schema.getFieldNames.foreach { name =>
mapping.put(name, name)
}
@@ -292,7 +292,7 @@ object SchemaValidator {
val schema = properties.getTableSchema(SCHEMA)
- schema.getColumnNames.zip(schema.getTypes).zipWithIndex.foreach { case ((n, t), i) =>
+ schema.getFieldNames.zip(schema.getFieldTypes).zipWithIndex.foreach { case ((n, t), i) =>
val isProctime = properties
.getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")
.orElse(false)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala
index 7943144..bbb9911 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/TemporalTableFunction.scala
@@ -74,7 +74,7 @@ object TemporalTableFunction {
timeAttribute,
primaryKey,
new RowTypeInfo(
- table.getSchema.getTypes,
- table.getSchema.getColumnNames))
+ table.getSchema.getFieldTypes,
+ table.getSchema.getFieldNames))
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
index e9195f6..0c3d25a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalTableSourceScan.scala
@@ -79,7 +79,7 @@ class FlinkLogicalTableSourceScan(
override def explainTerms(pw: RelWriter): RelWriter = {
val terms = super.explainTerms(pw)
- .item("fields", tableSource.getTableSchema.getColumnNames.mkString(", "))
+ .item("fields", tableSource.getTableSchema.getFieldNames.mkString(", "))
val sourceDesc = tableSource.explainSource()
if (sourceDesc.nonEmpty) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
index 849d16c..38cb197 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
@@ -90,7 +90,7 @@ abstract class CsvTableSinkFactoryBase extends TableFactory {
val csvTableSink = new CsvTableSink(path, fieldDelimiter)
csvTableSink
- .configure(formatSchema.getColumnNames, formatSchema.getTypes)
+ .configure(formatSchema.getFieldNames, formatSchema.getFieldTypes)
.asInstanceOf[CsvTableSink]
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
index d320220..c2cf8f5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
@@ -100,7 +100,7 @@ abstract class CsvTableSourceFactoryBase extends TableFactory {
toScala(params.getOptionalString(FORMAT_LINE_DELIMITER))
.foreach(csvTableSourceBuilder.lineDelimiter)
- formatSchema.getColumnNames.zip(formatSchema.getTypes).foreach { case (name, tpe) =>
+ formatSchema.getFieldNames.zip(formatSchema.getFieldTypes).foreach { case (name, tpe) =>
csvTableSourceBuilder.field(name, tpe)
}
toScala(params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSource.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSource.scala
index 1da1a0d..f92cd6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSource.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSource.scala
@@ -60,5 +60,5 @@ trait TableSource[T] {
* @return A String explaining the [[TableSource]].
*/
def explainSource(): String =
- TableConnectorUtil.generateRuntimeName(getClass, getTableSchema.getColumnNames)
+ TableConnectorUtil.generateRuntimeName(getClass, getTableSchema.getFieldNames)
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
index ef1aa16..15d6890 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
@@ -59,8 +59,8 @@ object TableSourceUtil {
def validateTableSource(tableSource: TableSource[_]): Unit = {
val schema = tableSource.getTableSchema
- val tableFieldNames = schema.getColumnNames
- val tableFieldTypes = schema.getTypes
+ val tableFieldNames = schema.getFieldNames
+ val tableFieldTypes = schema.getFieldTypes
// get rowtime and proctime attributes
val rowtimeAttributes = getRowtimeAttributes(tableSource)
@@ -111,14 +111,14 @@ object TableSourceUtil {
} else if (descriptors.size() == 1) {
val descriptor = descriptors.get(0)
val rowtimeAttribute = descriptor.getAttributeName
- val rowtimeIdx = schema.getColumnNames.indexOf(rowtimeAttribute)
+ val rowtimeIdx = schema.getFieldNames.indexOf(rowtimeAttribute)
// ensure that field exists
if (rowtimeIdx < 0) {
throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$rowtimeAttribute' but field '$rowtimeAttribute' does not exist in table.")
}
// ensure that field is of type TIMESTAMP
- if (schema.getTypes(rowtimeIdx) != Types.SQL_TIMESTAMP) {
+ if (schema.getFieldTypes()(rowtimeIdx) != Types.SQL_TIMESTAMP) {
throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$rowtimeAttribute' but field '$rowtimeAttribute' is not of type TIMESTAMP.")
}
@@ -135,14 +135,14 @@ object TableSourceUtil {
tableSource match {
case p: DefinedProctimeAttribute if p.getProctimeAttribute != null =>
val proctimeAttribute = p.getProctimeAttribute
- val proctimeIdx = schema.getColumnNames.indexOf(proctimeAttribute)
+ val proctimeIdx = schema.getFieldNames.indexOf(proctimeAttribute)
// ensure that field exists
if (proctimeIdx < 0) {
throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$proctimeAttribute' but field '$proctimeAttribute' does not exist in table.")
}
// ensure that field is of type TIMESTAMP
- if (schema.getTypes(proctimeIdx) != Types.SQL_TIMESTAMP) {
+ if (schema.getFieldTypes()(proctimeIdx) != Types.SQL_TIMESTAMP) {
throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$proctimeAttribute' but field '$proctimeAttribute' is not of type TIMESTAMP.")
}
@@ -177,18 +177,18 @@ object TableSourceUtil {
// get names of selected fields
val tableFieldNames = if (selectedFields.isDefined) {
- val names = tableSchema.getColumnNames
+ val names = tableSchema.getFieldNames
selectedFields.get.map(names(_))
} else {
- tableSchema.getColumnNames
+ tableSchema.getFieldNames
}
// get types of selected fields
val tableFieldTypes = if (selectedFields.isDefined) {
- val types = tableSchema.getTypes
+ val types = tableSchema.getFieldTypes
selectedFields.get.map(types(_))
} else {
- tableSchema.getTypes
+ tableSchema.getFieldTypes
}
// get rowtime and proctime attributes
@@ -255,8 +255,8 @@ object TableSourceUtil {
streaming: Boolean,
typeFactory: FlinkTypeFactory): RelDataType = {
- val fieldNames = tableSource.getTableSchema.getColumnNames
- var fieldTypes = tableSource.getTableSchema.getTypes
+ val fieldNames = tableSource.getTableSchema.getFieldNames
+ var fieldTypes = tableSource.getTableSchema.getFieldTypes
if (streaming) {
// adjust the type of time attributes for streaming tables
@@ -312,7 +312,7 @@ object TableSourceUtil {
} else {
val descriptor = descriptors.get(0)
// look up index of row time attribute in schema
- val fieldIdx = tableSource.getTableSchema.getColumnNames.indexOf(
+ val fieldIdx = tableSource.getTableSchema.getFieldNames.indexOf(
descriptor.getAttributeName)
// is field among selected fields?
if (selectedFields.get.contains(fieldIdx)) {
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableSchemaTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableSchemaTest.scala
index c8aca9d..eaa852f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableSchemaTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/TableSchemaTest.scala
@@ -32,20 +32,20 @@ class TableSchemaTest extends TableTestBase {
val table = util.addTable[(Int, String)]("MyTable", 'a, 'b)
val schema = table.getSchema
- assertEquals("a", schema.getColumnNames.apply(0))
- assertEquals("b", schema.getColumnNames.apply(1))
+ assertEquals("a", schema.getFieldNames.apply(0))
+ assertEquals("b", schema.getFieldNames.apply(1))
- assertEquals(Types.INT, schema.getTypes.apply(0))
- assertEquals(Types.STRING, schema.getTypes.apply(1))
+ assertEquals(Types.INT, schema.getFieldTypes.apply(0))
+ assertEquals(Types.STRING, schema.getFieldTypes.apply(1))
val expectedString = "root\n" +
" |-- a: Integer\n" +
" |-- b: String\n"
assertEquals(expectedString, schema.toString)
- assertTrue(schema.getColumnName(3).isEmpty)
- assertTrue(schema.getType(-1).isEmpty)
- assertTrue(schema.getType("c").isEmpty)
+ assertTrue(!schema.getFieldName(3).isPresent)
+ assertTrue(!schema.getFieldType(-1).isPresent)
+ assertTrue(!schema.getFieldType("c").isPresent)
}
@Test
@@ -54,19 +54,19 @@ class TableSchemaTest extends TableTestBase {
val table = util.addTable[(Int, String)]("MyTable", 'a, 'b)
val schema = table.getSchema
- assertEquals("a", schema.getColumnNames.apply(0))
- assertEquals("b", schema.getColumnNames.apply(1))
+ assertEquals("a", schema.getFieldNames.apply(0))
+ assertEquals("b", schema.getFieldNames.apply(1))
- assertEquals(Types.INT, schema.getTypes.apply(0))
- assertEquals(Types.STRING, schema.getTypes.apply(1))
+ assertEquals(Types.INT, schema.getFieldTypes.apply(0))
+ assertEquals(Types.STRING, schema.getFieldTypes.apply(1))
val expectedString = "root\n" +
" |-- a: Integer\n" +
" |-- b: String\n"
assertEquals(expectedString, schema.toString)
- assertTrue(schema.getColumnName(3).isEmpty)
- assertTrue(schema.getType(-1).isEmpty)
- assertTrue(schema.getType("c").isEmpty)
+ assertTrue(!schema.getFieldName(3).isPresent)
+ assertTrue(!schema.getFieldType(-1).isPresent)
+ assertTrue(!schema.getFieldType("c").isPresent)
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala
index ff510d3..023a5bd 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/batch/sql/validation/InsertIntoValidationTest.scala
@@ -65,7 +65,7 @@ class InsertIntoValidationTest extends TableTestBase {
util.addTable[(Int, Long, String)]("sourceTable", 'a, 'b, 'c)
val fieldNames = Array("d", "e", "f")
- val fieldTypes = util.tableEnv.scan("sourceTable").getSchema.getTypes
+ val fieldTypes = util.tableEnv.scan("sourceTable").getSchema.getFieldTypes
val sink = new MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink
util.tableEnv.registerTableSink("targetTable", fieldNames, fieldTypes, sink)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala
index 1343473..db5836d 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/sql/validation/InsertIntoValidationTest.scala
@@ -75,7 +75,7 @@ class InsertIntoValidationTest {
tEnv.registerTable("sourceTable", t)
val fieldNames = Array("d", "e", "f")
- val fieldTypes = tEnv.scan("sourceTable").getSchema.getTypes
+ val fieldTypes = tEnv.scan("sourceTable").getSchema.getFieldTypes
val sink = new MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink
tEnv.registerTableSink("targetTable", fieldNames, fieldTypes, sink)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala
index 82fa251..f8d4923 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/stream/table/TemporalTableJoinTest.scala
@@ -163,10 +163,10 @@ class TemporalTableJoinTest extends TableTestBase {
if (proctime) "proctime" else "rowtime",
rates.getTimeAttribute.asInstanceOf[ResolvedFieldReference].name)
assertArrayEquals(
- expectedSchema.getColumnNames.asInstanceOf[Array[Object]],
+ expectedSchema.getFieldNames.asInstanceOf[Array[Object]],
rates.getResultType.getFieldNames.asInstanceOf[Array[Object]])
assertArrayEquals(
- expectedSchema.getTypes.asInstanceOf[Array[Object]],
+ expectedSchema.getFieldTypes.asInstanceOf[Array[Object]],
rates.getResultType.getFieldTypes.asInstanceOf[Array[Object]])
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/validation/TableSchemaValidationTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/validation/TableSchemaValidationTest.scala
index c430e59..44b49f9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/validation/TableSchemaValidationTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/api/validation/TableSchemaValidationTest.scala
@@ -30,8 +30,8 @@ class TableSchemaValidationTest extends TableTestBase {
thrown.expectMessage(
"Number of field names and field types must be equal.\n" +
"Number of names is 3, number of types is 2.\n" +
- "List of field names: [a, b, c].\n" +
- "List of field types: [Integer, String].")
+ "List of field names: [a, b, c]\n" +
+ "List of field types: [Integer, String]")
val fieldNames = Array("a", "b", "c")
val typeInfos: Array[TypeInformation[_]] = Array(
@@ -45,8 +45,8 @@ class TableSchemaValidationTest extends TableTestBase {
thrown.expect(classOf[TableException])
thrown.expectMessage(
"Field names must be unique.\n" +
- "List of duplicate fields: [a].\n" +
- "List of all fields: [a, a, c].")
+ "List of duplicate fields: [a]\n" +
+ "List of all fields: [a, a, c]")
val fieldNames = Array("a", "a", "c")
val typeInfos: Array[TypeInformation[_]] = Array(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala
index 07839e6..51e31dc 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/sql/TableEnvironmentITCase.scala
@@ -126,7 +126,7 @@ class TableEnvironmentITCase(
tEnv.registerTable("sourceTable", t)
val fieldNames = Array("d", "e", "f")
- val fieldTypes = tEnv.scan("sourceTable").getSchema.getTypes
+ val fieldTypes = tEnv.scan("sourceTable").getSchema.getFieldTypes
val sink = new MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink
tEnv.registerTableSink("targetTable", fieldNames, fieldTypes, sink)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala
index 67fde38..a6ca563 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/batch/table/TableEnvironmentITCase.scala
@@ -192,7 +192,7 @@ class TableEnvironmentITCase(
tEnv.registerTable("sourceTable", t)
val fieldNames = Array("d", "e", "f")
- val fieldTypes = tEnv.scan("sourceTable").getSchema.getTypes
+ val fieldTypes = tEnv.scan("sourceTable").getSchema.getFieldTypes
val sink = new MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink
tEnv.registerTableSink("targetTable", fieldNames, fieldTypes, sink)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/InMemoryTableFactory.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/InMemoryTableFactory.scala
index 6b86c08..7d42935 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/InMemoryTableFactory.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/InMemoryTableFactory.scala
@@ -59,7 +59,7 @@ class InMemoryTableFactory(terminationCount: Int)
val tableSchema = SchemaValidator.deriveTableSinkSchema(params)
new MemoryTableSourceSinkUtil.UnsafeMemoryAppendTableSink()
- .configure(tableSchema.getColumnNames, tableSchema.getTypes)
+ .configure(tableSchema.getFieldNames, tableSchema.getFieldTypes)
.asInstanceOf[StreamTableSink[Row]]
}
@@ -81,7 +81,7 @@ class InMemoryTableFactory(terminationCount: Int)
// proctime
val proctimeAttributeOpt = SchemaValidator.deriveProctimeAttribute(params)
- val (names, types) = tableSchema.getColumnNames.zip(tableSchema.getTypes)
+ val (names, types) = tableSchema.getFieldNames.zip(tableSchema.getFieldTypes)
.filter(_._1 != proctimeAttributeOpt.get()).unzip
// rowtime
val rowtimeDescriptors = SchemaValidator.deriveRowtimeAttributes(params)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/testTableSources.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/testTableSources.scala
index f7263c8..bbc155b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/testTableSources.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/utils/testTableSources.scala
@@ -159,7 +159,7 @@ class TestNestedProjectableTableSource(
null)
with NestedFieldsProjectableTableSource[Row] {
- var readNestedFields: Seq[String] = tableSchema.getColumnNames.map(f => s"$f.*")
+ var readNestedFields: Seq[String] = tableSchema.getFieldNames.map(f => s"$f.*")
override def projectNestedFields(
fields: Array[Int],
[flink] 07/07: [FLINK-10687] [table] Move format descriptors and
validators to flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9e5a4272d22ecd8c56a877ea5ca6156c42e9c620
Author: Timo Walther <tw...@apache.org>
AuthorDate: Sat Oct 27 15:47:22 2018 +0200
[FLINK-10687] [table] Move format descriptors and validators to flink-table-common
This commit makes the flink-formats module Scala free by introducing a
flink-table-common module that is implemented in Java. This module contains
all classes that are required across different Maven modules.
Additionally, all classes in this module have been annotated with @Internal
and @PublicEvolving accordingly.
Since the methods in Descriptor were declared with private[flink] visibility,
they have been migrated to a new toProperties() method that is public.
This closes #6958.
---
docs/dev/table/sourceSinks.md | 20 ++-
.../flink-connector-elasticsearch-base/pom.xml | 9 ++
.../ElasticsearchUpsertTableSinkFactoryBase.java | 10 +-
.../flink/table/descriptors/Elasticsearch.java | 10 +-
.../table/descriptors/ElasticsearchValidator.java | 7 +-
...lasticsearchUpsertTableSinkFactoryTestBase.java | 3 +-
.../flink-connector-kafka-base/pom.xml | 9 ++
.../kafka/KafkaTableSourceSinkFactoryBase.java | 8 +-
.../org/apache/flink/table/descriptors/Kafka.java | 11 +-
.../flink/table/descriptors/KafkaValidator.java | 6 +-
.../kafka/KafkaJsonTableSourceFactoryTestBase.java | 8 +-
.../kafka/KafkaTableSourceSinkFactoryTestBase.java | 5 +-
flink-formats/flink-avro/pom.xml | 22 ++-
.../flink/formats/avro/AvroRowFormatFactory.java | 29 ++--
.../org/apache/flink/table/descriptors/Avro.java | 11 +-
.../flink/table/descriptors/AvroValidator.java | 2 +
.../formats/avro/AvroRowFormatFactoryTest.java | 14 +-
flink-formats/flink-json/pom.xml | 18 ++-
.../flink/formats/json/JsonRowFormatFactory.java | 52 +++----
.../org/apache/flink/table/descriptors/Json.java | 13 +-
.../flink/table/descriptors/JsonValidator.java | 6 +-
.../formats/json/JsonRowFormatFactoryTest.java | 6 +-
.../table/client/config/entries/FunctionEntry.java | 4 +-
.../gateway/utils/TestTableSinkFactoryBase.java | 2 +-
.../gateway/utils/TestTableSourceFactoryBase.java | 2 +-
.../table/descriptors/ConnectorDescriptor.java | 74 +++++++++
.../descriptors/ConnectorDescriptorValidator.java | 51 +++++++
.../Descriptor.java} | 26 ++--
.../flink/table/descriptors/DescriptorBase.java} | 21 +--
.../table/descriptors/DescriptorValidator.java} | 22 +--
.../flink/table/descriptors/FormatDescriptor.java | 60 ++++++++
.../descriptors/FormatDescriptorValidator.java | 61 ++++++++
.../factories/DeserializationSchemaFactory.java | 2 +
.../factories/SerializationSchemaFactory.java | 2 +
.../apache/flink/table/factories/TableFactory.java | 3 +
.../flink/table/factories/TableFormatFactory.java | 3 +
.../table/factories/TableFormatFactoryBase.java | 165 +++++++++++++++++++++
.../table/descriptors/DescriptorTestBase.java | 87 +++++++++++
.../factories/TableFormatFactoryBaseTest.java | 85 +++++++++++
flink-libraries/flink-table/pom.xml | 8 +
.../flink/table/catalog/ExternalCatalogTable.scala | 33 +++--
.../flink/table/descriptors/ClassInstance.scala | 11 +-
.../table/descriptors/ConnectTableDescriptor.scala | 19 ++-
.../table/descriptors/ConnectorDescriptor.scala | 55 -------
.../descriptors/ConnectorDescriptorValidator.scala | 53 -------
.../org/apache/flink/table/descriptors/Csv.scala | 14 +-
.../flink/table/descriptors/Descriptor.scala | 41 -----
.../flink/table/descriptors/FileSystem.scala | 17 ++-
.../flink/table/descriptors/FormatDescriptor.scala | 49 ------
.../descriptors/FormatDescriptorValidator.scala | 63 --------
.../table/descriptors/FunctionDescriptor.scala | 12 +-
.../flink/table/descriptors/LiteralValue.scala | 11 +-
.../apache/flink/table/descriptors/Metadata.scala | 11 +-
.../table/descriptors/RegistrableDescriptor.scala | 2 +-
.../apache/flink/table/descriptors/Rowtime.scala | 21 +--
.../apache/flink/table/descriptors/Schema.scala | 15 +-
.../flink/table/descriptors/SchemaValidator.scala | 32 ----
.../table/descriptors/SchematicDescriptor.scala | 2 +-
.../flink/table/descriptors/Statistics.scala | 13 +-
.../table/descriptors/StreamTableDescriptor.scala | 13 +-
.../flink/table/descriptors/TableDescriptor.scala | 4 +-
.../table/factories/TableFactoryService.scala | 6 +-
.../flink/table/factories/TableFactoryUtil.scala | 5 +-
.../flink/table/functions/FunctionService.scala | 8 +-
.../flink/table/util/JavaScalaConversionUtil.scala | 8 -
.../catalog/InMemoryExternalCatalogTest.scala | 10 +-
.../table/descriptors/DescriptorTestBase.scala | 130 ----------------
.../table/descriptors/SchemaValidatorTest.scala | 27 +---
.../table/descriptors/TableDescriptorTest.scala | 5 +-
.../table/descriptors/TestTableDescriptor.scala | 69 +++++++++
.../table/factories/utils/TestTableFormat.scala | 8 +-
.../factories/utils/TestTableFormatFactory.scala | 32 +---
.../flink/table/runtime/stream/sql/SqlITCase.scala | 14 +-
73 files changed, 1035 insertions(+), 735 deletions(-)
diff --git a/docs/dev/table/sourceSinks.md b/docs/dev/table/sourceSinks.md
index e80a680..d89c884 100644
--- a/docs/dev/table/sourceSinks.md
+++ b/docs/dev/table/sourceSinks.md
@@ -672,7 +672,8 @@ A connector for `MySystem` in our example can extend `ConnectorDescriptor` as sh
<div data-lang="java" markdown="1">
{% highlight java %}
import org.apache.flink.table.descriptors.ConnectorDescriptor;
-import org.apache.flink.table.descriptors.DescriptorProperties;
+import java.util.HashMap;
+import java.util.Map;
/**
* Connector to MySystem with debug mode.
@@ -687,8 +688,10 @@ public class MySystemConnector extends ConnectorDescriptor {
}
@Override
- public void addConnectorProperties(DescriptorProperties properties) {
- properties.putString("connector.debug", Boolean.toString(isDebug));
+ protected Map<String, String> toConnectorProperties() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("connector.debug", Boolean.toString(isDebug));
+ return properties;
}
}
{% endhighlight %}
@@ -697,15 +700,18 @@ public class MySystemConnector extends ConnectorDescriptor {
<div data-lang="scala" markdown="1">
{% highlight scala %}
import org.apache.flink.table.descriptors.ConnectorDescriptor
-import org.apache.flink.table.descriptors.DescriptorProperties
+import java.util.HashMap
+import java.util.Map
/**
* Connector to MySystem with debug mode.
*/
-class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system", 1, formatNeeded = false) {
+class MySystemConnector(isDebug: Boolean) extends ConnectorDescriptor("my-system", 1, false) {
- override protected def addConnectorProperties(properties: DescriptorProperties): Unit = {
- properties.putString("connector.debug", isDebug.toString)
+ override protected def toConnectorProperties(): Map[String, String] = {
+ val properties = new HashMap[String, String]
+ properties.put("connector.debug", isDebug.toString)
+ properties
}
}
{% endhighlight %}
diff --git a/flink-connectors/flink-connector-elasticsearch-base/pom.xml b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
index 9fbd9b3..b7b7940 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch-base/pom.xml
@@ -114,6 +114,15 @@ under the License.
<scope>test</scope>
</dependency>
+ <!-- Elasticsearch table descriptor testing -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<!-- Elasticsearch table sink factory testing -->
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryBase.java
index 79946de..63e9b34 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryBase.java
@@ -102,9 +102,9 @@ public abstract class ElasticsearchUpsertTableSinkFactoryBase implements StreamT
@Override
public Map<String, String> requiredContext() {
final Map<String, String> context = new HashMap<>();
- context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_ELASTICSEARCH);
- context.put(CONNECTOR_VERSION(), elasticsearchVersion());
- context.put(CONNECTOR_PROPERTY_VERSION(), "1");
+ context.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_ELASTICSEARCH);
+ context.put(CONNECTOR_VERSION, elasticsearchVersion());
+ context.put(CONNECTOR_PROPERTY_VERSION, "1");
return context;
}
@@ -140,7 +140,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryBase implements StreamT
properties.add(SCHEMA() + ".#." + SCHEMA_NAME());
// format wildcard
- properties.add(FORMAT() + ".*");
+ properties.add(FORMAT + ".*");
return properties;
}
@@ -210,7 +210,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryBase implements StreamT
}
private SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
- final String formatType = properties.get(FORMAT_TYPE());
+ final String formatType = properties.get(FORMAT_TYPE);
// we could have added this check to the table factory context
// but this approach allows to throw more helpful error messages
// if the supported format has not been added
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/Elasticsearch.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/Elasticsearch.java
index f306e19..fa07821 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/Elasticsearch.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/Elasticsearch.java
@@ -26,6 +26,7 @@ import org.apache.flink.util.Preconditions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION;
@@ -71,7 +72,7 @@ public class Elasticsearch extends ConnectorDescriptor {
* @param version Elasticsearch version. E.g., "6".
*/
public Elasticsearch version(String version) {
- internalProperties.putString(CONNECTOR_VERSION(), version);
+ internalProperties.putString(CONNECTOR_VERSION, version);
return this;
}
@@ -301,8 +302,9 @@ public class Elasticsearch extends ConnectorDescriptor {
}
@Override
- public void addConnectorProperties(DescriptorProperties properties) {
- properties.putProperties(internalProperties.asMap());
+ protected Map<String, String> toConnectorProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putProperties(internalProperties);
final List<List<String>> hostValues = hosts.stream()
.map(host -> Arrays.asList(host.hostname, String.valueOf(host.port), host.protocol))
@@ -311,5 +313,7 @@ public class Elasticsearch extends ConnectorDescriptor {
CONNECTOR_HOSTS,
Arrays.asList(CONNECTOR_HOSTS_HOSTNAME, CONNECTOR_HOSTS_PORT, CONNECTOR_HOSTS_PROTOCOL),
hostValues);
+
+ return properties.asMap();
}
}
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
index 1858ee1..9d1a926 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors;
+import org.apache.flink.annotation.Internal;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -29,6 +31,7 @@ import static org.apache.flink.table.descriptors.DescriptorProperties.noValidati
/**
* The validator for {@link Elasticsearch}.
*/
+@Internal
public class ElasticsearchValidator extends ConnectorDescriptorValidator {
public static final String CONNECTOR_TYPE_VALUE_ELASTICSEARCH = "elasticsearch";
@@ -63,7 +66,7 @@ public class ElasticsearchValidator extends ConnectorDescriptorValidator {
@Override
public void validate(DescriptorProperties properties) {
super.validate(properties);
- properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_ELASTICSEARCH, false);
+ properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_ELASTICSEARCH, false);
validateVersion(properties);
validateHosts(properties);
validateGeneralProperties(properties);
@@ -74,7 +77,7 @@ public class ElasticsearchValidator extends ConnectorDescriptorValidator {
private void validateVersion(DescriptorProperties properties) {
properties.validateEnumValues(
- CONNECTOR_VERSION(),
+ CONNECTOR_VERSION,
false,
Collections.singletonList(CONNECTOR_VERSION_VALUE_6));
}
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
index 88aee43..0cc282c 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
@@ -31,7 +31,6 @@ import org.apache.flink.table.descriptors.TestTableDescriptor;
import org.apache.flink.table.factories.StreamTableSinkFactory;
import org.apache.flink.table.factories.TableFactoryService;
import org.apache.flink.table.sinks.TableSink;
-import org.apache.flink.table.util.JavaScalaConversionUtil;
import org.apache.flink.types.Row;
import org.apache.flink.util.TestLogger;
@@ -113,7 +112,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryTestBase extends TestLo
.field(FIELD_TS, Types.SQL_TIMESTAMP()))
.inUpsertMode();
- final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = testDesc.toProperties();
final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
.createStreamTableSink(propertiesMap);
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index d6a6bb6..df98dd9 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -206,6 +206,15 @@ under the License.
<scope>test</scope>
</dependency>
+ <!-- Kafka table descriptor testing -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
index b4aea0f..72fcdca 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryBase.java
@@ -93,9 +93,9 @@ public abstract class KafkaTableSourceSinkFactoryBase implements
public Map<String, String> requiredContext() {
Map<String, String> context = new HashMap<>();
context.put(UPDATE_MODE(), UPDATE_MODE_VALUE_APPEND()); // append mode
- context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA); // kafka
- context.put(CONNECTOR_VERSION(), kafkaVersion()); // version
- context.put(CONNECTOR_PROPERTY_VERSION(), "1"); // backwards compatibility
+ context.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_KAFKA); // kafka
+ context.put(CONNECTOR_VERSION, kafkaVersion()); // version
+ context.put(CONNECTOR_PROPERTY_VERSION, "1"); // backwards compatibility
return context;
}
@@ -131,7 +131,7 @@ public abstract class KafkaTableSourceSinkFactoryBase implements
properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY());
// format wildcard
- properties.add(FORMAT() + ".*");
+ properties.add(FORMAT + ".*");
return properties;
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
index e44341a..125ece0 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
@@ -246,13 +246,12 @@ public class Kafka extends ConnectorDescriptor {
return this;
}
- /**
- * Internal method for connector properties conversion.
- */
@Override
- public void addConnectorProperties(DescriptorProperties properties) {
+ protected Map<String, String> toConnectorProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+
if (version != null) {
- properties.putString(CONNECTOR_VERSION(), version);
+ properties.putString(CONNECTOR_VERSION, version);
}
if (topic != null) {
@@ -290,5 +289,7 @@ public class Kafka extends ConnectorDescriptor {
properties.putClass(CONNECTOR_SINK_PARTITIONER_CLASS, sinkPartitionerClass);
}
}
+
+ return properties.asMap();
}
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
index 68939d2..86d7ef6 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
@@ -18,6 +18,7 @@
package org.apache.flink.table.descriptors;
+import org.apache.flink.annotation.Internal;
import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
import java.util.Arrays;
@@ -31,6 +32,7 @@ import static org.apache.flink.table.descriptors.DescriptorProperties.noValidati
/**
* The validator for {@link Kafka}.
*/
+@Internal
public class KafkaValidator extends ConnectorDescriptorValidator {
public static final String CONNECTOR_TYPE_VALUE_KAFKA = "kafka";
@@ -60,7 +62,7 @@ public class KafkaValidator extends ConnectorDescriptorValidator {
@Override
public void validate(DescriptorProperties properties) {
super.validate(properties);
- properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA, false);
+ properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE_KAFKA, false);
validateVersion(properties);
@@ -78,7 +80,7 @@ public class KafkaValidator extends ConnectorDescriptorValidator {
CONNECTOR_VERSION_VALUE_010,
CONNECTOR_VERSION_VALUE_011,
CONNECTOR_VERSION_VALUE_20);
- properties.validateEnumValues(CONNECTOR_VERSION(), false, versions);
+ properties.validateEnumValues(CONNECTOR_VERSION, false, versions);
properties.validateString(CONNECTOR_TOPIC, false, 1, Integer.MAX_VALUE);
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
index afb9517..28f4520 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
@@ -22,7 +22,6 @@ import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.formats.json.JsonRowSchemaConverter;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.descriptors.FormatDescriptor;
import org.apache.flink.table.descriptors.Json;
import org.apache.flink.table.descriptors.Kafka;
@@ -156,11 +155,10 @@ public abstract class KafkaJsonTableSourceFactoryTestBase {
.field("proc-time", Types.SQL_TIMESTAMP).proctime())
.inAppendMode();
- DescriptorProperties properties = new DescriptorProperties(true);
- testDesc.addProperties(properties);
+ final Map<String, String> properties = testDesc.toProperties();
final TableSource<?> factorySource =
- TableFactoryService.find(StreamTableSourceFactory.class, testDesc)
- .createStreamTableSource(properties.asMap());
+ TableFactoryService.find(StreamTableSourceFactory.class, properties)
+ .createStreamTableSource(properties);
assertEquals(builderSource, factorySource);
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
index 4beba49..f28b9f9 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
@@ -52,7 +52,6 @@ import org.apache.flink.table.sources.TableSource;
import org.apache.flink.table.sources.TableSourceUtil;
import org.apache.flink.table.sources.tsextractors.ExistingField;
import org.apache.flink.table.sources.wmstrategies.AscendingTimestamps;
-import org.apache.flink.table.util.JavaScalaConversionUtil;
import org.apache.flink.types.Row;
import org.apache.flink.util.TestLogger;
@@ -165,7 +164,7 @@ public abstract class KafkaTableSourceSinkFactoryTestBase extends TestLogger {
.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
.inAppendMode();
- final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = testDesc.toProperties();
final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
.createStreamTableSource(propertiesMap);
@@ -215,7 +214,7 @@ public abstract class KafkaTableSourceSinkFactoryTestBase extends TestLogger {
.field(EVENT_TIME, Types.SQL_TIMESTAMP()))
.inAppendMode();
- final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = testDesc.toProperties();
final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
.createStreamTableSink(propertiesMap);
diff --git a/flink-formats/flink-avro/pom.xml b/flink-formats/flink-avro/pom.xml
index 9d844c9..4f62a7e 100644
--- a/flink-formats/flink-avro/pom.xml
+++ b/flink-formats/flink-avro/pom.xml
@@ -64,18 +64,32 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <!-- use a dedicated Scala version to not depend on it -->
- <artifactId>flink-table_2.11</artifactId>
+ <artifactId>flink-table-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
- <!-- Projects depending on this project, won't depend on flink-table. -->
- <optional>true</optional>
</dependency>
<!-- test dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <!-- TODO This could be dropped if we move the Table Avro IT Case -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_2.11</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- TODO This could be dropped if we move the Table Avro IT Case -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-table_2.11</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroRowFormatFactory.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroRowFormatFactory.java
index 416dd82..e52ebdf 100644
--- a/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroRowFormatFactory.java
+++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/formats/avro/AvroRowFormatFactory.java
@@ -22,15 +22,14 @@ import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.table.descriptors.AvroValidator;
import org.apache.flink.table.descriptors.DescriptorProperties;
-import org.apache.flink.table.descriptors.FormatDescriptorValidator;
import org.apache.flink.table.factories.DeserializationSchemaFactory;
import org.apache.flink.table.factories.SerializationSchemaFactory;
+import org.apache.flink.table.factories.TableFormatFactoryBase;
import org.apache.flink.types.Row;
import org.apache.avro.specific.SpecificRecord;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,23 +37,15 @@ import java.util.Map;
* Table format factory for providing configured instances of Avro-to-row {@link SerializationSchema}
* and {@link DeserializationSchema}.
*/
-public class AvroRowFormatFactory implements SerializationSchemaFactory<Row>, DeserializationSchemaFactory<Row> {
+public class AvroRowFormatFactory extends TableFormatFactoryBase<Row>
+ implements SerializationSchemaFactory<Row>, DeserializationSchemaFactory<Row> {
- @Override
- public Map<String, String> requiredContext() {
- final Map<String, String> context = new HashMap<>();
- context.put(FormatDescriptorValidator.FORMAT_TYPE(), AvroValidator.FORMAT_TYPE_VALUE);
- context.put(FormatDescriptorValidator.FORMAT_PROPERTY_VERSION(), "1");
- return context;
- }
-
- @Override
- public boolean supportsSchemaDerivation() {
- return false;
+ public AvroRowFormatFactory() {
+ super(AvroValidator.FORMAT_TYPE_VALUE, 1, false);
}
@Override
- public List<String> supportedProperties() {
+ protected List<String> supportedFormatProperties() {
final List<String> properties = new ArrayList<>();
properties.add(AvroValidator.FORMAT_RECORD_CLASS);
properties.add(AvroValidator.FORMAT_AVRO_SCHEMA);
@@ -63,7 +54,7 @@ public class AvroRowFormatFactory implements SerializationSchemaFactory<Row>, De
@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
- final DescriptorProperties descriptorProperties = validateAndGetProperties(properties);
+ final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
// create and configure
if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
@@ -76,7 +67,7 @@ public class AvroRowFormatFactory implements SerializationSchemaFactory<Row>, De
@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
- final DescriptorProperties descriptorProperties = validateAndGetProperties(properties);
+ final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
// create and configure
if (descriptorProperties.containsKey(AvroValidator.FORMAT_RECORD_CLASS)) {
@@ -87,8 +78,8 @@ public class AvroRowFormatFactory implements SerializationSchemaFactory<Row>, De
}
}
- private static DescriptorProperties validateAndGetProperties(Map<String, String> propertiesMap) {
- final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
+ private static DescriptorProperties getValidatedProperties(Map<String, String> propertiesMap) {
+ final DescriptorProperties descriptorProperties = new DescriptorProperties();
descriptorProperties.putProperties(propertiesMap);
// validate
diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/Avro.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/Avro.java
index 611d714..f0ba8ac 100644
--- a/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/Avro.java
+++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/Avro.java
@@ -22,6 +22,8 @@ import org.apache.flink.util.Preconditions;
import org.apache.avro.specific.SpecificRecord;
+import java.util.Map;
+
/**
* Format descriptor for Apache Avro records.
*/
@@ -59,16 +61,17 @@ public class Avro extends FormatDescriptor {
return this;
}
- /**
- * Internal method for format properties conversion.
- */
@Override
- public void addFormatProperties(DescriptorProperties properties) {
+ protected Map<String, String> toFormatProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+
if (null != recordClass) {
properties.putClass(AvroValidator.FORMAT_RECORD_CLASS, recordClass);
}
if (null != avroSchema) {
properties.putString(AvroValidator.FORMAT_AVRO_SCHEMA, avroSchema);
}
+
+ return properties.asMap();
}
}
diff --git a/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/AvroValidator.java b/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/AvroValidator.java
index c66dcc7..cbee1aa 100644
--- a/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/AvroValidator.java
+++ b/flink-formats/flink-avro/src/main/java/org/apache/flink/table/descriptors/AvroValidator.java
@@ -18,11 +18,13 @@
package org.apache.flink.table.descriptors;
+import org.apache.flink.annotation.Internal;
import org.apache.flink.table.api.ValidationException;
/**
* Validator for {@link Avro}.
*/
+@Internal
public class AvroValidator extends FormatDescriptorValidator {
public static final String FORMAT_TYPE_VALUE = "avro";
diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroRowFormatFactoryTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroRowFormatFactoryTest.java
index 26fdd53..15a8dbd 100644
--- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroRowFormatFactoryTest.java
+++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroRowFormatFactoryTest.java
@@ -22,8 +22,6 @@ import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.formats.avro.generated.User;
import org.apache.flink.table.descriptors.Avro;
-import org.apache.flink.table.descriptors.Descriptor;
-import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.factories.DeserializationSchemaFactory;
import org.apache.flink.table.factories.SerializationSchemaFactory;
import org.apache.flink.table.factories.TableFactoryService;
@@ -46,7 +44,7 @@ public class AvroRowFormatFactoryTest extends TestLogger {
@Test
public void testRecordClass() {
- final Map<String, String> properties = toMap(new Avro().recordClass(AVRO_SPECIFIC_RECORD));
+ final Map<String, String> properties = new Avro().recordClass(AVRO_SPECIFIC_RECORD).toProperties();
testRecordClassDeserializationSchema(properties);
@@ -55,7 +53,7 @@ public class AvroRowFormatFactoryTest extends TestLogger {
@Test
public void testAvroSchema() {
- final Map<String, String> properties = toMap(new Avro().avroSchema(AVRO_SCHEMA));
+ final Map<String, String> properties = new Avro().avroSchema(AVRO_SCHEMA).toProperties();
testAvroSchemaSerializationSchema(properties);
@@ -93,12 +91,4 @@ public class AvroRowFormatFactoryTest extends TestLogger {
final SerializationSchema<?> expected1 = new AvroRowSerializationSchema(AVRO_SCHEMA);
assertEquals(expected1, actual1);
}
-
- private static Map<String, String> toMap(Descriptor... desc) {
- final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
- for (Descriptor d : desc) {
- d.addProperties(descriptorProperties);
- }
- return descriptorProperties.asMap();
- }
}
diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml
index 0cb3f8d..4b1baa9 100644
--- a/flink-formats/flink-json/pom.xml
+++ b/flink-formats/flink-json/pom.xml
@@ -53,12 +53,9 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <!-- use a dedicated Scala version to not depend on it -->
- <artifactId>flink-table_2.11</artifactId>
+ <artifactId>flink-table-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
- <!-- Projects depending on this project, won't depend on flink-table. -->
- <optional>true</optional>
</dependency>
<!-- test dependencies -->
@@ -68,16 +65,25 @@ under the License.
<artifactId>flink-test-utils-junit</artifactId>
</dependency>
+ <!-- JSON table descriptor testing -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <!-- TODO This could be dropped if we change JsonRowFormatFactoryTest -->
<dependency>
<groupId>org.apache.flink</groupId>
<!-- use a dedicated Scala version to not depend on it -->
<artifactId>flink-table_2.11</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- flink-table needs Scala -->
+ <!-- TODO This could be dropped if we change JsonRowFormatFactoryTest -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowFormatFactory.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowFormatFactory.java
index 458b94a..567bef3 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowFormatFactory.java
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowFormatFactory.java
@@ -23,15 +23,13 @@ import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.table.descriptors.DescriptorProperties;
-import org.apache.flink.table.descriptors.FormatDescriptorValidator;
import org.apache.flink.table.descriptors.JsonValidator;
-import org.apache.flink.table.descriptors.SchemaValidator;
import org.apache.flink.table.factories.DeserializationSchemaFactory;
import org.apache.flink.table.factories.SerializationSchemaFactory;
+import org.apache.flink.table.factories.TableFormatFactoryBase;
import org.apache.flink.types.Row;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,35 +37,25 @@ import java.util.Map;
* Table format factory for providing configured instances of JSON-to-row {@link SerializationSchema}
* and {@link DeserializationSchema}.
*/
-public class JsonRowFormatFactory implements SerializationSchemaFactory<Row>, DeserializationSchemaFactory<Row> {
+public class JsonRowFormatFactory extends TableFormatFactoryBase<Row>
+ implements SerializationSchemaFactory<Row>, DeserializationSchemaFactory<Row> {
- @Override
- public Map<String, String> requiredContext() {
- final Map<String, String> context = new HashMap<>();
- context.put(FormatDescriptorValidator.FORMAT_TYPE(), JsonValidator.FORMAT_TYPE_VALUE);
- context.put(FormatDescriptorValidator.FORMAT_PROPERTY_VERSION(), "1");
- return context;
+ public JsonRowFormatFactory() {
+ super(JsonValidator.FORMAT_TYPE_VALUE, 1, true);
}
@Override
- public boolean supportsSchemaDerivation() {
- return true;
- }
-
- @Override
- public List<String> supportedProperties() {
+ protected List<String> supportedFormatProperties() {
final List<String> properties = new ArrayList<>();
properties.add(JsonValidator.FORMAT_JSON_SCHEMA);
properties.add(JsonValidator.FORMAT_SCHEMA);
properties.add(JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD);
- properties.add(FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA());
- properties.addAll(SchemaValidator.getSchemaDerivationKeys());
return properties;
}
@Override
public DeserializationSchema<Row> createDeserializationSchema(Map<String, String> properties) {
- final DescriptorProperties descriptorProperties = validateAndGetProperties(properties);
+ final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
// create and configure
final JsonRowDeserializationSchema schema = new JsonRowDeserializationSchema(createTypeInformation(descriptorProperties));
@@ -80,29 +68,29 @@ public class JsonRowFormatFactory implements SerializationSchemaFactory<Row>, De
@Override
public SerializationSchema<Row> createSerializationSchema(Map<String, String> properties) {
- final DescriptorProperties descriptorProperties = validateAndGetProperties(properties);
+ final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
// create and configure
return new JsonRowSerializationSchema(createTypeInformation(descriptorProperties));
}
- private static DescriptorProperties validateAndGetProperties(Map<String, String> propertiesMap) {
- final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
- descriptorProperties.putProperties(propertiesMap);
-
- // validate
- new JsonValidator().validate(descriptorProperties);
-
- return descriptorProperties;
- }
-
- private static TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
+ private TypeInformation<Row> createTypeInformation(DescriptorProperties descriptorProperties) {
if (descriptorProperties.containsKey(JsonValidator.FORMAT_SCHEMA)) {
return (RowTypeInfo) descriptorProperties.getType(JsonValidator.FORMAT_SCHEMA);
} else if (descriptorProperties.containsKey(JsonValidator.FORMAT_JSON_SCHEMA)) {
return JsonRowSchemaConverter.convert(descriptorProperties.getString(JsonValidator.FORMAT_JSON_SCHEMA));
} else {
- return SchemaValidator.deriveFormatFields(descriptorProperties).toRowType();
+ return deriveSchema(descriptorProperties.asMap()).toRowType();
}
}
+
+ private static DescriptorProperties getValidatedProperties(Map<String, String> propertiesMap) {
+ final DescriptorProperties descriptorProperties = new DescriptorProperties();
+ descriptorProperties.putProperties(propertiesMap);
+
+ // validate
+ new JsonValidator().validate(descriptorProperties);
+
+ return descriptorProperties;
+ }
}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
index 8e5b6f3..3c4888c 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
@@ -23,6 +23,8 @@ import org.apache.flink.table.utils.TypeStringUtils;
import org.apache.flink.types.Row;
import org.apache.flink.util.Preconditions;
+import java.util.Map;
+
import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA;
import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD;
import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_JSON_SCHEMA;
@@ -106,13 +108,12 @@ public class Json extends FormatDescriptor {
return this;
}
- /**
- * Internal method for format properties conversion.
- */
@Override
- public void addFormatProperties(DescriptorProperties properties) {
+ protected Map<String, String> toFormatProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+
if (deriveSchema != null) {
- properties.putBoolean(FORMAT_DERIVE_SCHEMA(), deriveSchema);
+ properties.putBoolean(FORMAT_DERIVE_SCHEMA, deriveSchema);
}
if (jsonSchema != null) {
@@ -126,5 +127,7 @@ public class Json extends FormatDescriptor {
if (failOnMissingField != null) {
properties.putBoolean(FORMAT_FAIL_ON_MISSING_FIELD, failOnMissingField);
}
+
+ return properties.asMap();
}
}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
index af8721f..a520811 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
@@ -18,11 +18,13 @@
package org.apache.flink.table.descriptors;
+import org.apache.flink.annotation.Internal;
import org.apache.flink.table.api.ValidationException;
/**
* Validator for {@link Json}.
*/
+@Internal
public class JsonValidator extends FormatDescriptorValidator {
public static final String FORMAT_TYPE_VALUE = "json";
@@ -33,8 +35,8 @@ public class JsonValidator extends FormatDescriptorValidator {
@Override
public void validate(DescriptorProperties properties) {
super.validate(properties);
- properties.validateBoolean(FORMAT_DERIVE_SCHEMA(), true);
- final boolean deriveSchema = properties.getOptionalBoolean(FORMAT_DERIVE_SCHEMA()).orElse(false);
+ properties.validateBoolean(FORMAT_DERIVE_SCHEMA, true);
+ final boolean deriveSchema = properties.getOptionalBoolean(FORMAT_DERIVE_SCHEMA).orElse(false);
final boolean hasSchema = properties.containsKey(FORMAT_SCHEMA);
final boolean hasSchemaString = properties.containsKey(FORMAT_JSON_SCHEMA);
if (deriveSchema && (hasSchema || hasSchemaString)) {
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowFormatFactoryTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowFormatFactoryTest.java
index f43234d..caf99f4 100644
--- a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowFormatFactoryTest.java
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowFormatFactoryTest.java
@@ -119,7 +119,7 @@ public class JsonRowFormatFactoryTest extends TestLogger {
final SerializationSchema<?> actual1 = TableFactoryService
.find(SerializationSchemaFactory.class, properties)
.createSerializationSchema(properties);
- final SerializationSchema expected1 = new JsonRowSerializationSchema(SCHEMA);
+ final SerializationSchema<?> expected1 = new JsonRowSerializationSchema(SCHEMA);
assertEquals(expected1, actual1);
}
@@ -141,9 +141,9 @@ public class JsonRowFormatFactoryTest extends TestLogger {
}
private static Map<String, String> toMap(Descriptor... desc) {
- final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
+ final DescriptorProperties descriptorProperties = new DescriptorProperties();
for (Descriptor d : desc) {
- d.addProperties(descriptorProperties);
+ descriptorProperties.putProperties(d.toProperties());
}
return descriptorProperties.asMap();
}
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/FunctionEntry.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/FunctionEntry.java
index 1b006cb..e05d10b 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/FunctionEntry.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/config/entries/FunctionEntry.java
@@ -73,8 +73,8 @@ public class FunctionEntry extends ConfigEntry {
private class FunctionEntryDescriptor extends FunctionDescriptor {
@Override
- public void addProperties(DescriptorProperties properties) {
- properties.putProperties(FunctionEntry.this.properties);
+ public Map<String, String> toProperties() {
+ return FunctionEntry.this.properties.asMap();
}
}
}
diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
index 1de2407..d7d4f9f 100644
--- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
+++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSinkFactoryBase.java
@@ -62,7 +62,7 @@ public abstract class TestTableSinkFactoryBase implements StreamTableSinkFactory
public Map<String, String> requiredContext() {
final Map<String, String> context = new HashMap<>();
context.put(UPDATE_MODE(), UPDATE_MODE_VALUE_APPEND());
- context.put(CONNECTOR_TYPE(), type);
+ context.put(CONNECTOR_TYPE, type);
return context;
}
diff --git a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
index f6ad16d..59091d8 100644
--- a/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
+++ b/flink-libraries/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/utils/TestTableSourceFactoryBase.java
@@ -65,7 +65,7 @@ public abstract class TestTableSourceFactoryBase implements StreamTableSourceFac
public Map<String, String> requiredContext() {
final Map<String, String> context = new HashMap<>();
context.put(UPDATE_MODE(), UPDATE_MODE_VALUE_APPEND());
- context.put(CONNECTOR_TYPE(), type);
+ context.put(CONNECTOR_TYPE, type);
return context;
}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java
new file mode 100644
index 0000000..a1d3e78
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Map;
+
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION;
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE;
+
+/**
+ * Describes a connector to an other system.
+ */
+@PublicEvolving
+public abstract class ConnectorDescriptor extends DescriptorBase implements Descriptor {
+
+ private String type;
+
+ private int version;
+
+ private boolean formatNeeded;
+
+ /**
+ * Constructs a {@link ConnectorDescriptor}.
+ *
+ * @param type string that identifies this connector
+ * @param version property version for backwards compatibility
+ * @param formatNeeded flag for basic validation of a needed format descriptor
+ */
+ public ConnectorDescriptor(String type, int version, boolean formatNeeded) {
+ this.type = type;
+ this.version = version;
+ this.formatNeeded = formatNeeded;
+ }
+
+ @Override
+ public final Map<String, String> toProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putString(CONNECTOR_TYPE, type);
+ properties.putLong(CONNECTOR_PROPERTY_VERSION, version);
+ properties.putProperties(toConnectorProperties());
+ return properties.asMap();
+ }
+
+ /**
+ * Returns if this connector requires a format descriptor.
+ */
+ protected final boolean isFormatNeeded() {
+ return formatNeeded;
+ }
+
+ /**
+ * Converts this descriptor into a set of connector properties. Usually prefixed with
+ * {@link FormatDescriptorValidator#FORMAT}.
+ */
+ protected abstract Map<String, String> toConnectorProperties();
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java
new file mode 100644
index 0000000..88cf349
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Validator for {@link ConnectorDescriptor}.
+ */
+@Internal
+public abstract class ConnectorDescriptorValidator implements DescriptorValidator {
+
+ /**
+ * Key for describing the type of the connector. Usually used for factory discovery.
+ */
+ public static final String CONNECTOR_TYPE = "connector.type";
+
+ /**
+ * Key for describing the property version. This property can be used for backwards
+ * compatibility in case the property format changes.
+ */
+ public static final String CONNECTOR_PROPERTY_VERSION = "connector.property-version";
+
+ /**
+ * Key for describing the version of the connector. This property can be used for different
+ * connector versions (e.g. Kafka 0.8 or Kafka 0.11).
+ */
+ public static final String CONNECTOR_VERSION = "connector.version";
+
+ @Override
+ public void validate(DescriptorProperties properties) {
+ properties.validateString(CONNECTOR_TYPE, false, 1);
+ properties.validateInt(CONNECTOR_PROPERTY_VERSION, true, 0);
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/Descriptor.java
similarity index 55%
copy from flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
copy to flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/Descriptor.java
index 20c8296..caa7149 100644
--- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/Descriptor.java
@@ -16,26 +16,28 @@
* limitations under the License.
*/
-package org.apache.flink.table.factories;
+package org.apache.flink.table.descriptors;
-import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.annotation.PublicEvolving;
import java.util.Map;
/**
- * Factory for creating configured instances of {@link SerializationSchema}.
+ * Interface that adds a set of string-based, normalized properties for describing DDL information.
*
- * @param <T> record type that the format produces or consumes.
+ * <p>Typical characteristics of a descriptor are:
+ * - descriptors have a default constructor
+ * - descriptors themselves contain very little logic
+ * - corresponding validators validate the correctness (goal: have a single point of validation)
+ *
+ * <p>A descriptor is similar to a builder in a builder pattern, thus, mutable for building
+ * properties.
*/
-public interface SerializationSchemaFactory<T> extends TableFormatFactory<T> {
+@PublicEvolving
+public interface Descriptor {
/**
- * Creates and configures a [[SerializationSchema]] using the given properties.
- *
- * @param properties normalized properties describing the format
- * @return the configured serialization schema or null if the factory cannot provide an
- * instance of this class
+ * Converts this descriptor into a set of properties.
*/
- SerializationSchema<T> createSerializationSchema(Map<String, String> properties);
-
+ Map<String, String> toProperties();
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorBase.java
similarity index 70%
copy from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala
copy to flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorBase.java
index 007a406..355d046 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorBase.java
@@ -16,17 +16,18 @@
* limitations under the License.
*/
-package org.apache.flink.table.descriptors
+package org.apache.flink.table.descriptors;
-/**
- * Validator for a descriptor. We put the validation methods and utilities in separate classes
- * to keep the descriptor interfaces clean.
- */
-trait DescriptorValidator {
+import org.apache.flink.annotation.PublicEvolving;
- /**
- * Performs basic validation such as completeness tests.
- */
- def validate(properties: DescriptorProperties): Unit
+/**
+ * Base class for {@link Descriptor}s.
+ */
+@PublicEvolving
+public abstract class DescriptorBase implements Descriptor {
+ @Override
+ public String toString() {
+ return DescriptorProperties.toString(toProperties());
+ }
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorValidator.java
similarity index 67%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala
rename to flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorValidator.java
index 007a406..14e1be7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorValidator.scala
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorValidator.java
@@ -16,17 +16,19 @@
* limitations under the License.
*/
-package org.apache.flink.table.descriptors
+package org.apache.flink.table.descriptors;
-/**
- * Validator for a descriptor. We put the validation methods and utilities in separate classes
- * to keep the descriptor interfaces clean.
- */
-trait DescriptorValidator {
+import org.apache.flink.annotation.Internal;
- /**
- * Performs basic validation such as completeness tests.
- */
- def validate(properties: DescriptorProperties): Unit
+/**
+ * Validator for a descriptor. We put the validation methods and utilities in separate classes
+ * to keep the descriptor interfaces clean.
+ */
+@Internal
+public interface DescriptorValidator {
+ /**
+ * Performs basic validation such as completeness tests.
+ */
+ void validate(DescriptorProperties properties);
}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptor.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptor.java
new file mode 100644
index 0000000..89039ce
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Map;
+
+/**
+ * Describes the format of data.
+ */
+@PublicEvolving
+public abstract class FormatDescriptor extends DescriptorBase implements Descriptor {
+
+ private String type;
+
+ private int version;
+
+ /**
+ * Constructs a {@link FormatDescriptor}.
+ *
+ * @param type string that identifies this format
+ * @param version property version for backwards compatibility
+ */
+ public FormatDescriptor(String type, int version) {
+ this.type = type;
+ this.version = version;
+ }
+
+ @Override
+ public final Map<String, String> toProperties() {
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putString(FormatDescriptorValidator.FORMAT_TYPE, type);
+ properties.putInt(FormatDescriptorValidator.FORMAT_PROPERTY_VERSION, version);
+ properties.putProperties(toFormatProperties());
+ return properties.asMap();
+ }
+
+ /**
+ * Converts this descriptor into a set of format properties. Usually prefixed with
+ * {@link FormatDescriptorValidator#FORMAT}.
+ */
+ protected abstract Map<String, String> toFormatProperties();
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptorValidator.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptorValidator.java
new file mode 100644
index 0000000..f5c90b7
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/FormatDescriptorValidator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Validator for {@link FormatDescriptor}.
+ */
+@Internal
+public abstract class FormatDescriptorValidator implements DescriptorValidator {
+
+ /**
+ * Prefix for format-related properties.
+ */
+ public static final String FORMAT = "format";
+
+ /**
+ * Key for describing the type of the format. Usually used for factory discovery.
+ */
+ public static final String FORMAT_TYPE = "format.type";
+
+ /**
+ * Key for describing the property version. This property can be used for backwards
+ * compatibility in case the property format changes.
+ */
+ public static final String FORMAT_PROPERTY_VERSION = "format.property-version";
+
+ /**
+ * Key for describing the version of the format. This property can be used for different
+ * format versions (e.g. Avro 1.8.2 or Avro 2.0).
+ */
+ public static final String FORMAT_VERSION = "format.version";
+
+ /**
+ * Key for deriving the schema of the format from the table's schema.
+ */
+ public static final String FORMAT_DERIVE_SCHEMA = "format.derive-schema";
+
+ @Override
+ public void validate(DescriptorProperties properties) {
+ properties.validateString(FORMAT_TYPE, false, 1);
+ properties.validateString(FORMAT_PROPERTY_VERSION, true, 1);
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
index 12af6ce..5dab1ee 100644
--- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
@@ -18,6 +18,7 @@
package org.apache.flink.table.factories;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import java.util.Map;
@@ -27,6 +28,7 @@ import java.util.Map;
*
* @param <T> record type that the format produces or consumes.
*/
+@PublicEvolving
public interface DeserializationSchemaFactory<T> extends TableFormatFactory<T> {
/**
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
index 20c8296..b7b1718 100644
--- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
@@ -18,6 +18,7 @@
package org.apache.flink.table.factories;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.SerializationSchema;
import java.util.Map;
@@ -27,6 +28,7 @@ import java.util.Map;
*
* @param <T> record type that the format produces or consumes.
*/
+@PublicEvolving
public interface SerializationSchemaFactory<T> extends TableFormatFactory<T> {
/**
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
index 647feb5..b56ffa1 100644
--- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
@@ -18,6 +18,8 @@
package org.apache.flink.table.factories;
+import org.apache.flink.annotation.PublicEvolving;
+
import java.util.List;
import java.util.Map;
@@ -33,6 +35,7 @@ import java.util.Map;
*
* @see TableFormatFactory
*/
+@PublicEvolving
public interface TableFactory {
/**
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
index da298f2..dce5315 100644
--- a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
@@ -18,6 +18,8 @@
package org.apache.flink.table.factories;
+import org.apache.flink.annotation.PublicEvolving;
+
import java.util.List;
/**
@@ -29,6 +31,7 @@ import java.util.List;
*
* @param <T> record type that the format produces or consumes.
*/
+@PublicEvolving
public interface TableFormatFactory<T> extends TableFactory {
/**
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
new file mode 100644
index 0000000..07e5921
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactoryBase.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.factories;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.descriptors.DescriptorProperties;
+import org.apache.flink.table.descriptors.FormatDescriptorValidator;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Base class for {@link TableFormatFactory}s.
+ *
+ * @param <T> record type that the format produces or consumes.
+ */
+@PublicEvolving
+public abstract class TableFormatFactoryBase<T> implements TableFormatFactory<T> {
+
+ // Constants for schema derivation
+ // TODO drop constants once SchemaValidator has been ported to flink-table-common
+ private static final String SCHEMA = "schema";
+ private static final String SCHEMA_NAME = "name";
+ private static final String SCHEMA_TYPE = "type";
+ private static final String SCHEMA_PROCTIME = "proctime";
+ private static final String SCHEMA_FROM = "from";
+ private static final String ROWTIME_TIMESTAMPS_TYPE = "rowtime.timestamps.type";
+ private static final String ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD = "from-field";
+ private static final String ROWTIME_TIMESTAMPS_FROM = "rowtime.timestamps.from";
+ private static final String ROWTIME_TIMESTAMPS_CLASS = "rowtime.timestamps.class";
+ private static final String ROWTIME_TIMESTAMPS_SERIALIZED = "rowtime.timestamps.serialized";
+ private static final String ROWTIME_WATERMARKS_TYPE = "rowtime.watermarks.type";
+ private static final String ROWTIME_WATERMARKS_CLASS = "rowtime.watermarks.class";
+ private static final String ROWTIME_WATERMARKS_SERIALIZED = "rowtime.watermarks.serialized";
+ private static final String ROWTIME_WATERMARKS_DELAY = "rowtime.watermarks.delay";
+
+ private String type;
+
+ private String version;
+
+ private boolean supportsSchemaDerivation;
+
+ public TableFormatFactoryBase(String type, int version, boolean supportsSchemaDerivation) {
+ this.type = type;
+ this.version = Integer.toString(version);
+ this.supportsSchemaDerivation = supportsSchemaDerivation;
+ }
+
+ @Override
+ public final Map<String, String> requiredContext() {
+ final Map<String, String> context = new HashMap<>();
+ context.put(FormatDescriptorValidator.FORMAT_TYPE, type);
+ context.put(FormatDescriptorValidator.FORMAT_PROPERTY_VERSION, version);
+ context.putAll(requiredFormatContext());
+ return context;
+ }
+
+ @Override
+ public final boolean supportsSchemaDerivation() {
+ return supportsSchemaDerivation;
+ }
+
+ @Override
+ public final List<String> supportedProperties() {
+ final List<String> properties = new ArrayList<>();
+ if (supportsSchemaDerivation) {
+ properties.add(FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA);
+ // schema
+ properties.add(SCHEMA + ".#." + SCHEMA_TYPE);
+ properties.add(SCHEMA + ".#." + SCHEMA_NAME);
+ properties.add(SCHEMA + ".#." + SCHEMA_FROM);
+ // time attributes
+ properties.add(SCHEMA + ".#." + SCHEMA_PROCTIME);
+ properties.add(SCHEMA + ".#." + ROWTIME_TIMESTAMPS_TYPE);
+ properties.add(SCHEMA + ".#." + ROWTIME_TIMESTAMPS_FROM);
+ properties.add(SCHEMA + ".#." + ROWTIME_TIMESTAMPS_CLASS);
+ properties.add(SCHEMA + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED);
+ properties.add(SCHEMA + ".#." + ROWTIME_WATERMARKS_TYPE);
+ properties.add(SCHEMA + ".#." + ROWTIME_WATERMARKS_CLASS);
+ properties.add(SCHEMA + ".#." + ROWTIME_WATERMARKS_SERIALIZED);
+ properties.add(SCHEMA + ".#." + ROWTIME_WATERMARKS_DELAY);
+ }
+ properties.addAll(supportedFormatProperties());
+ return properties;
+ }
+
+ /**
+ * Format specific context.
+ *
+ * <p>This method can be used if format type and a property version is not enough.
+ */
+ protected Map<String, String> requiredFormatContext() {
+ return Collections.emptyMap();
+ }
+
+ /**
+ * Format specific supported properties.
+ *
+ * <p>This method can be used if schema derivation is not enough.
+ */
+ protected List<String> supportedFormatProperties() {
+ return Collections.emptyList();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Finds the table schema that can be used for a format schema (without time attributes).
+ */
+ public static TableSchema deriveSchema(Map<String, String> properties) {
+ final DescriptorProperties descriptorProperties = new DescriptorProperties();
+ descriptorProperties.putProperties(properties);
+
+ final TableSchema.Builder builder = TableSchema.builder();
+
+ final TableSchema baseSchema = descriptorProperties.getTableSchema(SCHEMA);
+ for (int i = 0; i < baseSchema.getFieldCount(); i++) {
+ final String fieldName = baseSchema.getFieldNames()[i];
+ final TypeInformation<?> fieldType = baseSchema.getFieldTypes()[i];
+
+ final boolean isProctime = descriptorProperties
+ .getOptionalBoolean(SCHEMA + '.' + i + '.' + SCHEMA_PROCTIME)
+ .orElse(false);
+ final String timestampKey = SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_TYPE;
+ final boolean isRowtime = descriptorProperties.containsKey(timestampKey);
+ if (!isProctime && !isRowtime) {
+ // check for aliasing
+ final String aliasName = descriptorProperties
+ .getOptionalString(SCHEMA + '.' + i + '.' + SCHEMA_FROM)
+ .orElse(fieldName);
+ builder.field(aliasName, fieldType);
+ }
+ // only use the rowtime attribute if it references a field
+ else if (isRowtime &&
+ descriptorProperties.isValue(timestampKey, ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD)) {
+ final String aliasName = descriptorProperties
+ .getString(SCHEMA + '.' + i + '.' + ROWTIME_TIMESTAMPS_FROM);
+ builder.field(aliasName, fieldType);
+ }
+ }
+
+ return builder.build();
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/descriptors/DescriptorTestBase.java b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/descriptors/DescriptorTestBase.java
new file mode 100644
index 0000000..8cb0933
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/descriptors/DescriptorTestBase.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.util.Preconditions;
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test base for testing {@link Descriptor} together with {@link DescriptorValidator}.
+ */
+public abstract class DescriptorTestBase {
+
+ /**
+ * Returns a set of valid descriptors.
+ */
+ protected abstract List<Descriptor> descriptors();
+
+ /**
+ * Returns a set of properties for each valid descriptor.
+ */
+ protected abstract List<Map<String, String>> properties();
+
+ /**
+ * Returns a validator that can validate all valid descriptors.
+ */
+ protected abstract DescriptorValidator validator();
+
+ @Test
+ public void testValidation() {
+ final List<Descriptor> descriptors = descriptors();
+ final List<Map<String, String>> properties = properties();
+
+ Preconditions.checkArgument(descriptors.size() == properties.size());
+
+ for (int i = 0; i < descriptors.size(); i++) {
+ verifyProperties(descriptors.get(i), properties.get(i));
+ }
+ }
+
+ protected void verifyProperties(Descriptor descriptor, Map<String, String> expected) {
+ // test produced properties
+ assertEquals(expected, descriptor.toProperties());
+
+ // test validation logic
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putProperties(expected);
+ validator().validate(properties);
+ }
+
+ protected void addPropertyAndVerify(Descriptor descriptor, String property, String newValue) {
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putProperties(descriptor.toProperties());
+ final DescriptorProperties copy = properties.withoutKeys(Collections.singletonList(property));
+ copy.putString(property, newValue);
+ validator().validate(copy);
+ }
+
+ protected void removePropertyAndVerify(Descriptor descriptor, String property) {
+ final DescriptorProperties properties = new DescriptorProperties();
+ properties.putProperties(descriptor.toProperties());
+ final DescriptorProperties copy = properties.withoutKeys(Collections.singletonList(property));
+ validator().validate(copy);
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/factories/TableFormatFactoryBaseTest.java b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/factories/TableFormatFactoryBaseTest.java
new file mode 100644
index 0000000..b38eba3
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/factories/TableFormatFactoryBaseTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.factories;
+
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.table.api.TableSchema;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link TableFormatFactoryBase}.
+ */
+public class TableFormatFactoryBaseTest {
+
+ @Test
+ public void testSchemaDerivation() {
+ final Map<String, String> properties = new HashMap<>();
+ properties.put("schema.0.name", "otherField");
+ properties.put("schema.0.type", "VARCHAR");
+ properties.put("schema.0.from", "csvField");
+ properties.put("schema.1.name", "abcField");
+ properties.put("schema.1.type", "VARCHAR");
+ properties.put("schema.2.name", "p");
+ properties.put("schema.2.type", "TIMESTAMP");
+ properties.put("schema.2.proctime", "true");
+ properties.put("schema.3.name", "r");
+ properties.put("schema.3.type", "TIMESTAMP");
+ properties.put("schema.3.rowtime.timestamps.type", "from-source");
+ properties.put("schema.3.rowtime.watermarks.type", "from-source");
+
+ final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties);
+ final TableSchema expectedSchema = TableSchema.builder()
+ .field("csvField", Types.STRING) // aliased
+ .field("abcField", Types.STRING)
+ .build();
+ assertEquals(expectedSchema, actualSchema);
+ }
+
+ @Test
+ public void testSchemaDerivationWithRowtime() {
+ final Map<String, String> properties = new HashMap<>();
+ properties.put("schema.0.name", "otherField");
+ properties.put("schema.0.type", "VARCHAR");
+ properties.put("schema.0.from", "csvField");
+ properties.put("schema.1.name", "abcField");
+ properties.put("schema.1.type", "VARCHAR");
+ properties.put("schema.2.name", "p");
+ properties.put("schema.2.type", "TIMESTAMP");
+ properties.put("schema.2.proctime", "true");
+ properties.put("schema.3.name", "r");
+ properties.put("schema.3.type", "TIMESTAMP");
+ properties.put("schema.3.rowtime.timestamps.type", "from-field"); // from-field strategy
+ properties.put("schema.3.rowtime.timestamps.from", "myTime");
+ properties.put("schema.3.rowtime.watermarks.type", "from-source");
+
+ final TableSchema actualSchema = TableFormatFactoryBase.deriveSchema(properties);
+ final TableSchema expectedSchema = TableSchema.builder()
+ .field("csvField", Types.STRING) // aliased
+ .field("abcField", Types.STRING)
+ .field("myTime", Types.SQL_TIMESTAMP)
+ .build();
+ assertEquals(expectedSchema, actualSchema);
+ }
+}
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index c7b3c55..536f9c9 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -156,6 +156,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-test-utils_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
index cf5857e..45414ee 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
@@ -18,12 +18,13 @@
package org.apache.flink.table.catalog
+import java.util
+
import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, readColumnStats}
import org.apache.flink.table.descriptors.StreamTableDescriptorValidator.{UPDATE_MODE, UPDATE_MODE_VALUE_APPEND, UPDATE_MODE_VALUE_RETRACT, UPDATE_MODE_VALUE_UPSERT}
import org.apache.flink.table.descriptors._
import org.apache.flink.table.factories.TableFactory
import org.apache.flink.table.plan.stats.TableStats
-import org.apache.flink.table.util.JavaScalaConversionUtil
import org.apache.flink.table.util.JavaScalaConversionUtil.toScala
import scala.collection.JavaConverters._
@@ -63,7 +64,7 @@ class ExternalCatalogTable(
@deprecated
def getTableStats: Option[TableStats] = {
val normalizedProps = new DescriptorProperties()
- addProperties(normalizedProps)
+ normalizedProps.putProperties(normalizedProps)
val rowCount = toScala(normalizedProps.getOptionalLong(STATISTICS_ROW_COUNT))
rowCount match {
case Some(cnt) =>
@@ -109,10 +110,10 @@ class ExternalCatalogTable(
// ----------------------------------------------------------------------------------------------
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override private[flink] def addProperties(descriptorProperties: DescriptorProperties): Unit = {
- descriptorProperties.putProperties(properties)
+ override def toProperties: util.Map[String, String] = {
+ properties
}
}
@@ -295,7 +296,7 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = true,
isSink = false,
- JavaScalaConversionUtil.toJavaMap(this))
+ toProperties)
}
/**
@@ -310,7 +311,7 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = false,
isSink = true,
- JavaScalaConversionUtil.toJavaMap(this))
+ toProperties)
}
/**
@@ -325,20 +326,22 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = true,
isSink = true,
- JavaScalaConversionUtil.toJavaMap(this))
+ toProperties)
}
// ----------------------------------------------------------------------------------------------
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- connectorDescriptor.addProperties(properties)
- formatDescriptor.foreach(_.addProperties(properties))
- schemaDescriptor.foreach(_.addProperties(properties))
- statisticsDescriptor.foreach(_.addProperties(properties))
- metadataDescriptor.foreach(_.addProperties(properties))
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+ properties.putProperties(connectorDescriptor.toProperties)
+ formatDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ schemaDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ statisticsDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ metadataDescriptor.foreach(d => properties.putProperties(d.toProperties))
updateMode.foreach(mode => properties.putString(UPDATE_MODE, mode))
+ properties.asMap()
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstance.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstance.scala
index 007f2ba..34ed6dd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstance.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstance.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.Types
@@ -194,10 +196,12 @@ class ClassInstance extends HierarchyDescriptor {
}
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
addPropertiesWithPrefix(HierarchyDescriptorValidator.EMPTY_PREFIX, properties)
+ properties.asMap()
}
/**
@@ -233,6 +237,9 @@ object ClassInstance {
/**
* Descriptor for a class instance. A class instance is a Java/Scala object created from a class
* with a public constructor (with or without parameters).
+ *
+ * @deprecated Use `new ClassInstance()`.
*/
+ @deprecated
def apply() = new ClassInstance
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectTableDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectTableDescriptor.scala
index 569b825..3152c11 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectTableDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectTableDescriptor.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.api.{TableEnvironment, ValidationException}
import org.apache.flink.table.factories.TableFactoryUtil
@@ -86,23 +88,26 @@ abstract class ConnectTableDescriptor[D <: ConnectTableDescriptor[D]](
// ----------------------------------------------------------------------------------------------
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
// this performs only basic validation
// more validation can only happen within a factory
- if (connectorDescriptor.needsFormat() && formatDescriptor.isEmpty) {
+ if (connectorDescriptor.isFormatNeeded && formatDescriptor.isEmpty) {
throw new ValidationException(
s"The connector '$connectorDescriptor' requires a format description.")
- } else if (!connectorDescriptor.needsFormat() && formatDescriptor.isDefined) {
+ } else if (!connectorDescriptor.isFormatNeeded && formatDescriptor.isDefined) {
throw new ValidationException(
s"The connector '$connectorDescriptor' does not require a format description " +
s"but '${formatDescriptor.get}' found.")
}
- connectorDescriptor.addProperties(properties)
- formatDescriptor.foreach(_.addProperties(properties))
- schemaDescriptor.foreach(_.addProperties(properties))
+ properties.putProperties(connectorDescriptor.toProperties)
+ formatDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ schemaDescriptor.foreach(d => properties.putProperties(d.toProperties))
+
+ properties.asMap()
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
deleted file mode 100644
index dc344f3..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
-
-/**
- * Describes a connector to an other system.
- *
- * @param tpe string identifier for the connector
- */
-abstract class ConnectorDescriptor(
- private val tpe: String,
- private val version: Int,
- private val formatNeeded: Boolean)
- extends Descriptor {
-
- override def toString: String = this.getClass.getSimpleName
-
- /**
- * Internal method for properties conversion.
- */
- final private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- properties.putString(CONNECTOR_TYPE, tpe)
- properties.putLong(CONNECTOR_PROPERTY_VERSION, version)
- addConnectorProperties(properties)
- }
-
- /**
- * Internal method for connector properties conversion.
- */
- protected def addConnectorProperties(properties: DescriptorProperties): Unit
-
- /**
- * Internal method that defines if this connector requires a format descriptor.
- */
- private[flink] def needsFormat(): Boolean = formatNeeded
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
deleted file mode 100644
index 2b24c82..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
-
-/**
- * Validator for [[ConnectorDescriptor]].
- */
-class ConnectorDescriptorValidator extends DescriptorValidator {
-
- override def validate(properties: DescriptorProperties): Unit = {
- properties.validateString(CONNECTOR_TYPE, false, 1)
- properties.validateInt(CONNECTOR_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
- }
-}
-
-object ConnectorDescriptorValidator {
-
- /**
- * Key for describing the type of the connector. Usually used for factory discovery.
- */
- val CONNECTOR_TYPE = "connector.type"
-
- /**
- * Key for describing the property version. This property can be used for backwards
- * compatibility in case the property format changes.
- */
- val CONNECTOR_PROPERTY_VERSION = "connector.property-version"
-
- /**
- * Key for describing the version of the connector. This property can be used for different
- * connector versions (e.g. Kafka 0.8 or Kafka 0.11).
- */
- val CONNECTOR_VERSION = "connector.version"
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
index e1204a1..6e63931 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
@@ -31,7 +31,7 @@ import scala.collection.JavaConverters._
/**
* Format descriptor for comma-separated values (CSV).
*/
-class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
+class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, 1) {
private var fieldDelim: Option[String] = None
private var lineDelim: Option[String] = None
@@ -143,10 +143,9 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
this
}
- /**
- * Internal method for format properties conversion.
- */
- override protected def addFormatProperties(properties: DescriptorProperties): Unit = {
+ override protected def toFormatProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
fieldDelim.foreach(properties.putString(FORMAT_FIELD_DELIMITER, _))
lineDelim.foreach(properties.putString(FORMAT_LINE_DELIMITER, _))
@@ -164,6 +163,8 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
commentPrefix.foreach(properties.putString(FORMAT_COMMENT_PREFIX, _))
isIgnoreFirstLine.foreach(properties.putBoolean(FORMAT_IGNORE_FIRST_LINE, _))
lenient.foreach(properties.putBoolean(FORMAT_IGNORE_PARSE_ERRORS, _))
+
+ properties.asMap()
}
}
@@ -174,7 +175,10 @@ object Csv {
/**
* Format descriptor for comma-separated values (CSV).
+ *
+ * @deprecated Use `new Csv()`.
*/
+ @deprecated
def apply(): Csv = new Csv()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Descriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Descriptor.scala
deleted file mode 100644
index aa96bc4..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Descriptor.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-/**
- * A trait that adds a set of string-based, normalized properties for describing DDL information.
- *
- * Typical characteristics of a descriptor are:
- * - descriptors have a default constructor and a default 'apply()' method for Scala
- * - descriptors themselves contain very little logic
- * - corresponding validators validate the correctness (goal: have a single point of validation)
- */
-trait Descriptor {
-
- /**
- * Internal method for properties conversion.
- */
- private[flink] def addProperties(properties: DescriptorProperties): Unit
-
- override def toString: String = {
- val descriptorProperties = new DescriptorProperties()
- addProperties(descriptorProperties)
- descriptorProperties.toString
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
index f306b5a..77cf27b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
@@ -18,13 +18,14 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, CONNECTOR_TYPE_VALUE}
/**
* Connector descriptor for a file system.
*/
-class FileSystem extends ConnectorDescriptor(
- CONNECTOR_TYPE_VALUE, version = 1, formatNeeded = true) {
+class FileSystem extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, 1, true) {
private var path: Option[String] = None
@@ -38,11 +39,12 @@ class FileSystem extends ConnectorDescriptor(
this
}
- /**
- * Internal method for properties conversion.
- */
- override protected def addConnectorProperties(properties: DescriptorProperties): Unit = {
+ override protected def toConnectorProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
path.foreach(properties.putString(CONNECTOR_PATH, _))
+
+ properties.asMap()
}
}
@@ -53,7 +55,10 @@ object FileSystem {
/**
* Connector descriptor for a file system.
+ *
+ * @deprecated Use `new FileSystem()`.
*/
+ @deprecated
def apply(): FileSystem = new FileSystem()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
deleted file mode 100644
index bca67c6..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION}
-
-/**
- * Describes the format of data.
- *
- * @param tpe string identifier for the format
- */
-abstract class FormatDescriptor(
- private val tpe: String,
- private val version: Int)
- extends Descriptor {
-
- override def toString: String = this.getClass.getSimpleName
-
- /**
- * Internal method for properties conversion.
- */
- final private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- properties.putString(FORMAT_TYPE, tpe)
- properties.putInt(FORMAT_PROPERTY_VERSION, version)
- addFormatProperties(properties)
- }
-
- /**
- * Internal method for format properties conversion.
- */
- protected def addFormatProperties(properties: DescriptorProperties): Unit
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
deleted file mode 100644
index 5c37c0a..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE}
-
-/**
- * Validator for [[FormatDescriptor]].
- */
-class FormatDescriptorValidator extends DescriptorValidator {
-
- override def validate(properties: DescriptorProperties): Unit = {
- properties.validateString(FORMAT_TYPE, false, 1)
- properties.validateInt(FORMAT_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
- }
-}
-
-object FormatDescriptorValidator {
-
- /**
- * Prefix for format-related properties.
- */
- val FORMAT = "format"
-
- /**
- * Key for describing the type of the format. Usually used for factory discovery.
- */
- val FORMAT_TYPE = "format.type"
-
- /**
- * Key for describing the property version. This property can be used for backwards
- * compatibility in case the property format changes.
- */
- val FORMAT_PROPERTY_VERSION = "format.property-version"
-
- /**
- * Key for describing the version of the format. This property can be used for different
- * format versions (e.g. Avro 1.8.2 or Avro 2.0).
- */
- val FORMAT_VERSION = "format.version"
-
- /**
- * Key for deriving the schema of the format from the table's schema.
- */
- val FORMAT_DERIVE_SCHEMA = "format.derive-schema"
-
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptor.scala
index 13d5056..59aef11 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptor.scala
@@ -17,6 +17,7 @@
*/
package org.apache.flink.table.descriptors
+import java.util
/**
* Descriptor for describing a function.
@@ -36,11 +37,13 @@ class FunctionDescriptor extends Descriptor {
}
/**
- * Internal method for format properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override def addProperties(properties: DescriptorProperties): Unit = {
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
from.foreach(properties.putString(FunctionDescriptorValidator.FROM, _))
- classInstance.foreach(_.addProperties(properties))
+ classInstance.foreach(d => properties.putProperties(d.toProperties))
+ properties.asMap()
}
}
@@ -51,6 +54,9 @@ object FunctionDescriptor {
/**
* Descriptor for describing a function.
+ *
+ * @deprecated Use `new FunctionDescriptor()`.
*/
+ @deprecated
def apply(): FunctionDescriptor = new FunctionDescriptor()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
index 8f3ff77..62bd6f8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.ValidationException
import org.apache.flink.table.utils.TypeStringUtils
@@ -156,10 +158,12 @@ class LiteralValue extends HierarchyDescriptor {
}
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
addPropertiesWithPrefix(HierarchyDescriptorValidator.EMPTY_PREFIX, properties)
+ properties.asMap()
}
/**
@@ -216,6 +220,9 @@ object LiteralValue {
* - "42", "-5" -> INT
* - "2.0", "1234.222" -> DOUBLE
* - VARCHAR otherwise
+ *
+ * @deprecated Use `new Literal()`.
*/
+ @deprecated
def apply() = new LiteralValue()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Metadata.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Metadata.scala
index 211d786..e5dfb79 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Metadata.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Metadata.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME}
/**
@@ -60,12 +62,14 @@ class Metadata extends Descriptor {
}
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- final override def addProperties(properties: DescriptorProperties): Unit = {
+ final override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
comment.foreach(c => properties.putString(METADATA_COMMENT, c))
creationTime.foreach(t => properties.putLong(METADATA_CREATION_TIME, t))
lastAccessTime.foreach(t => properties.putLong(METADATA_LAST_ACCESS_TIME, t))
+ properties.asMap()
}
}
@@ -76,6 +80,9 @@ object Metadata {
/**
* Metadata descriptor for adding additional, useful information.
+ *
+ * @deprecated Use `new Metadata()`.
*/
+ @deprecated
def apply(): Metadata = new Metadata()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RegistrableDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RegistrableDescriptor.scala
index e89ca8c..62a2851 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RegistrableDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RegistrableDescriptor.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.descriptors
/**
* A trait for descriptors that allow to register table source and/or sinks.
*/
-trait RegistrableDescriptor extends TableDescriptor {
+trait RegistrableDescriptor extends Descriptor {
/**
* Searches for the specified table source, configures it accordingly, and registers it as
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
index d928396..b56df4d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
@@ -23,9 +23,6 @@ import org.apache.flink.table.descriptors.RowtimeValidator.{normalizeTimestampEx
import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor}
import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy}
-import scala.collection.mutable
-import scala.collection.JavaConverters._
-
/**
* Rowtime descriptor for describing an event time attribute in the schema.
*/
@@ -109,14 +106,18 @@ class Rowtime extends Descriptor {
this
}
- /**
- * Internal method for properties conversion.
+ /**
+ * Converts this descriptor into a set of properties.
*/
- final override def addProperties(properties: DescriptorProperties): Unit = {
- val props = mutable.HashMap[String, String]()
- timestampExtractor.foreach(normalizeTimestampExtractor(_).foreach(e => props.put(e._1, e._2)))
- watermarkStrategy.foreach(normalizeWatermarkStrategy(_).foreach(e => props.put(e._1, e._2)))
- properties.putProperties(props.toMap.asJava)
+ final override def toProperties: java.util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
+ timestampExtractor.foreach(normalizeTimestampExtractor(_)
+ .foreach(e => properties.putString(e._1, e._2)))
+ watermarkStrategy.foreach(normalizeWatermarkStrategy(_)
+ .foreach(e => properties.putString(e._1, e._2)))
+
+ properties.asMap()
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
index f4224b9..4916c9d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.{TableSchema, ValidationException}
import org.apache.flink.table.descriptors.SchemaValidator._
@@ -131,24 +133,24 @@ class Schema extends Descriptor {
lastField match {
case None => throw new ValidationException("No field defined previously. Use field() before.")
case Some(f) =>
- val fieldProperties = new DescriptorProperties()
- rowtime.addProperties(fieldProperties)
- tableSchema(f) ++= fieldProperties.asMap.asScala
+ tableSchema(f) ++= rowtime.toProperties.asScala
lastField = None
}
this
}
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- final override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
+ final override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
properties.putIndexedVariableProperties(
SCHEMA,
tableSchema.toSeq.map { case (name, props) =>
(Map(SCHEMA_NAME -> name) ++ props).asJava
}.asJava
)
+ properties.asMap()
}
}
@@ -159,6 +161,9 @@ object Schema {
/**
* Describes a schema of a table.
+ *
+ * @deprecated Use `new Schema()`.
*/
+ @deprecated
def apply(): Schema = new Schema()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
index 8c8230a..5078727 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
@@ -282,36 +282,4 @@ object SchemaValidator {
mapping.toMap.asJava
}
-
- /**
- * Finds the fields that can be used for a format schema (without time attributes).
- */
- def deriveFormatFields(properties: DescriptorProperties): TableSchema = {
-
- val builder = TableSchema.builder()
-
- val schema = properties.getTableSchema(SCHEMA)
-
- schema.getFieldNames.zip(schema.getFieldTypes).zipWithIndex.foreach { case ((n, t), i) =>
- val isProctime = properties
- .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")
- .orElse(false)
- val tsType = s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE"
- val isRowtime = properties.containsKey(tsType)
- if (!isProctime && !isRowtime) {
- // check for a aliasing
- val fieldName = properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM")
- .orElse(n)
- builder.field(fieldName, t)
- }
- // only use the rowtime attribute if it references a field
- else if (isRowtime &&
- properties.getString(tsType) == ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD) {
- val field = properties.getString(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_FROM")
- builder.field(field, t)
- }
- }
-
- builder.build()
- }
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala
index 794ff9e..d1501bf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchematicDescriptor.scala
@@ -21,7 +21,7 @@ package org.apache.flink.table.descriptors
/**
* A trait for descriptors that allow to define a format and schema.
*/
-trait SchematicDescriptor[D <: SchematicDescriptor[D]] extends TableDescriptor {
+trait SchematicDescriptor[D <: SchematicDescriptor[D]] extends Descriptor {
/**
* Specifies the format that defines how to read data from a connector.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
index f87a868..1707e1b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.descriptors.StatisticsValidator._
import org.apache.flink.table.plan.stats.{ColumnStats, TableStats}
@@ -132,9 +134,11 @@ class Statistics extends Descriptor {
}
/**
- * Internal method for properties conversion.
+ * Converts this descriptor into a set of properties.
*/
- final override def addProperties(properties: DescriptorProperties): Unit = {
+ final override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
properties.putInt(STATISTICS_PROPERTY_VERSION, 1)
rowCount.foreach(rc => properties.putLong(STATISTICS_ROW_COUNT, rc))
val namedStats = columnStats.map { case (name, stats) =>
@@ -142,6 +146,8 @@ class Statistics extends Descriptor {
(stats + (NAME -> name)).toMap.asJava
}.toList.asJava
properties.putIndexedVariableProperties(STATISTICS_COLUMNS, namedStats)
+
+ properties.asMap()
}
}
@@ -152,6 +158,9 @@ object Statistics {
/**
* Statistics descriptor for describing table stats.
+ *
+ * @deprecated Use `new Statistics()`.
*/
+ @deprecated
def apply(): Statistics = new Statistics()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptor.scala
index 24c3302..cfda4d9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptor.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.api.StreamTableEnvironment
import org.apache.flink.table.descriptors.StreamTableDescriptorValidator._
@@ -87,11 +89,12 @@ class StreamTableDescriptor(
// ----------------------------------------------------------------------------------------------
- /**
- * Internal method for properties conversion.
- */
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- super.addProperties(properties)
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
+ properties.putProperties(super.toProperties)
updateMode.foreach(mode => properties.putString(UPDATE_MODE, mode))
+
+ properties.asMap()
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableDescriptor.scala
index b14a310..32a5cfe 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableDescriptor.scala
@@ -19,6 +19,6 @@
package org.apache.flink.table.descriptors
/**
- * Common trait for all descriptors describing table sources and sinks.
+ * Common class for all descriptors describing table sources and sinks.
*/
-trait TableDescriptor extends Descriptor
+abstract class TableDescriptor extends DescriptorBase
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
index da7c9cd..10b6a67 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
@@ -26,7 +26,7 @@ import org.apache.flink.table.descriptors.FormatDescriptorValidator._
import org.apache.flink.table.descriptors.MetadataValidator._
import org.apache.flink.table.descriptors.StatisticsValidator._
import org.apache.flink.table.descriptors._
-import org.apache.flink.table.util.{JavaScalaConversionUtil, Logging}
+import org.apache.flink.table.util.Logging
import org.apache.flink.util.Preconditions
import _root_.scala.collection.JavaConverters._
@@ -50,7 +50,7 @@ object TableFactoryService extends Logging {
def find[T](factoryClass: Class[T], descriptor: Descriptor): T = {
Preconditions.checkNotNull(descriptor)
- findInternal(factoryClass, JavaScalaConversionUtil.toJavaMap(descriptor), None)
+ findInternal(factoryClass, descriptor.toProperties, None)
}
/**
@@ -66,7 +66,7 @@ object TableFactoryService extends Logging {
Preconditions.checkNotNull(descriptor)
Preconditions.checkNotNull(classLoader)
- findInternal(factoryClass, JavaScalaConversionUtil.toJavaMap(descriptor), Some(classLoader))
+ findInternal(factoryClass, descriptor.toProperties, Some(classLoader))
}
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
index 1656ac7..b773684 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
@@ -22,7 +22,6 @@ import org.apache.flink.table.api.{BatchTableEnvironment, StreamTableEnvironment
import org.apache.flink.table.descriptors.Descriptor
import org.apache.flink.table.sinks.TableSink
import org.apache.flink.table.sources.TableSource
-import org.apache.flink.table.util.JavaScalaConversionUtil
/**
* Utility for dealing with [[TableFactory]] using the [[TableFactoryService]].
@@ -37,7 +36,7 @@ object TableFactoryUtil {
descriptor: Descriptor)
: TableSource[T] = {
- val javaMap = JavaScalaConversionUtil.toJavaMap(descriptor)
+ val javaMap = descriptor.toProperties
tableEnvironment match {
case _: BatchTableEnvironment =>
@@ -63,7 +62,7 @@ object TableFactoryUtil {
descriptor: Descriptor)
: TableSink[T] = {
- val javaMap = JavaScalaConversionUtil.toJavaMap(descriptor)
+ val javaMap = descriptor.toProperties
tableEnvironment match {
case _: BatchTableEnvironment =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/FunctionService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/FunctionService.scala
index d1d9552..acc1a1c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/FunctionService.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/FunctionService.scala
@@ -56,18 +56,18 @@ object FunctionService extends Logging {
performValidation: Boolean = true)
: UserDefinedFunction = {
- val descriptorProperties = new DescriptorProperties(true)
- descriptor.addProperties(descriptorProperties)
+ val properties = new DescriptorProperties(true)
+ properties.putProperties(descriptor.toProperties)
// validate
if (performValidation) {
- new FunctionDescriptorValidator().validate(descriptorProperties)
+ new FunctionDescriptorValidator().validate(properties)
}
// instantiate
val (instanceClass, instance) = generateInstance[AnyRef](
HierarchyDescriptorValidator.EMPTY_PREFIX,
- descriptorProperties,
+ properties,
classLoader)
if (!classOf[UserDefinedFunction].isAssignableFrom(instanceClass)) {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
index 3a60dbc..6dc79ed 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
@@ -18,12 +18,10 @@
package org.apache.flink.table.util
-import java.util
import java.util.Optional
import java.util.function.{BiConsumer, Consumer, Function}
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
-import org.apache.flink.table.descriptors.{Descriptor, DescriptorProperties}
/**
* Utilities for interoperability between Scala and Java classes.
@@ -60,10 +58,4 @@ object JavaScalaConversionUtil {
def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
new JTuple2[T0, T1](tuple._1, tuple._2)
}
-
- def toJavaMap(descriptor: Descriptor): util.Map[String, String] = {
- val descriptorProperties = new DescriptorProperties()
- descriptor.addProperties(descriptorProperties)
- descriptorProperties.asMap
- }
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/catalog/InMemoryExternalCatalogTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/catalog/InMemoryExternalCatalogTest.scala
index 1f84b3d..5238bfe 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/catalog/InMemoryExternalCatalogTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/catalog/InMemoryExternalCatalogTest.scala
@@ -20,9 +20,9 @@ package org.apache.flink.table.catalog
import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
import org.apache.flink.table.api._
-import org.apache.flink.table.descriptors.{ConnectorDescriptor, DescriptorProperties, Schema}
-import org.junit.{Before, Test}
+import org.apache.flink.table.descriptors.{ConnectorDescriptor, Schema}
import org.junit.Assert._
+import org.junit.{Before, Test}
class InMemoryExternalCatalogTest {
@@ -156,9 +156,9 @@ class InMemoryExternalCatalogTest {
.asTableSource()
}
- class TestConnectorDesc extends ConnectorDescriptor("test", version = 1, formatNeeded = false) {
- override protected def addConnectorProperties(properties: DescriptorProperties): Unit = {
+ class TestConnectorDesc extends ConnectorDescriptor("test", 1, false) {
+ override protected def toConnectorProperties: _root_.java.util.Map[String, String] = {
+ _root_.java.util.Collections.emptyMap()
}
}
-
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
deleted file mode 100644
index c062cd9..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import java.util.Collections
-
-import org.apache.flink.table.descriptors.StreamTableDescriptorValidator.{UPDATE_MODE, UPDATE_MODE_VALUE_APPEND, UPDATE_MODE_VALUE_RETRACT, UPDATE_MODE_VALUE_UPSERT}
-import org.apache.flink.util.Preconditions
-import org.junit.Assert.assertEquals
-import org.junit.Test
-
-import scala.collection.JavaConverters._
-
-abstract class DescriptorTestBase {
-
- /**
- * Returns a set of valid descriptors.
- * This method is implemented in both Scala and Java.
- */
- def descriptors(): java.util.List[Descriptor]
-
- /**
- * Returns a set of properties for each valid descriptor.
- * This code is implemented in both Scala and Java.
- */
- def properties(): java.util.List[java.util.Map[String, String]]
-
- /**
- * Returns a validator that can validate all valid descriptors.
- */
- def validator(): DescriptorValidator
-
- @Test
- def testValidation(): Unit = {
- val d = descriptors().asScala
- val p = properties().asScala
-
- Preconditions.checkArgument(d.length == p.length)
-
- d.zip(p).foreach { case (desc, props) =>
- verifyProperties(desc, props.asScala.toMap)
- }
- }
-
- def verifyProperties(descriptor: Descriptor, expected: Map[String, String]): Unit = {
- val normProps = new DescriptorProperties
- descriptor.addProperties(normProps)
-
- // test produced properties
- assertEquals(expected, normProps.asMap.asScala.toMap)
-
- // test validation logic
- validator().validate(normProps)
- }
-
- def addPropertyAndVerify(
- descriptor: Descriptor,
- property: String,
- invalidValue: String): Unit = {
- val properties = new DescriptorProperties
- descriptor.addProperties(properties)
- val copy = properties.withoutKeys(Collections.singletonList(property))
- copy.putString(property, invalidValue)
- validator().validate(copy)
- }
-
- def removePropertyAndVerify(descriptor: Descriptor, removeProperty: String): Unit = {
- val properties = new DescriptorProperties
- descriptor.addProperties(properties)
- validator().validate(properties.withoutKeys(Collections.singletonList(removeProperty)))
- }
-}
-
-class TestTableDescriptor(connector: ConnectorDescriptor)
- extends TableDescriptor
- with SchematicDescriptor[TestTableDescriptor]
- with StreamableDescriptor[TestTableDescriptor] {
-
- private var formatDescriptor: Option[FormatDescriptor] = None
- private var schemaDescriptor: Option[Schema] = None
- private var updateMode: Option[String] = None
-
- override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- connector.addProperties(properties)
- formatDescriptor.foreach(_.addProperties(properties))
- schemaDescriptor.foreach(_.addProperties(properties))
- updateMode.foreach(mode => properties.putString(UPDATE_MODE, mode))
- }
-
- override def withFormat(format: FormatDescriptor): TestTableDescriptor = {
- this.formatDescriptor = Some(format)
- this
- }
-
- override def withSchema(schema: Schema): TestTableDescriptor = {
- this.schemaDescriptor = Some(schema)
- this
- }
-
- override def inAppendMode(): TestTableDescriptor = {
- updateMode = Some(UPDATE_MODE_VALUE_APPEND)
- this
- }
-
- override def inRetractMode(): TestTableDescriptor = {
- updateMode = Some(UPDATE_MODE_VALUE_RETRACT)
- this
- }
-
- override def inUpsertMode(): TestTableDescriptor = {
- updateMode = Some(UPDATE_MODE_VALUE_UPSERT)
- this
- }
-}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
index b5c664f..2769cdf 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
@@ -43,7 +43,7 @@ class SchemaValidatorTest {
.field("r", Types.SQL_TIMESTAMP).rowtime(
Rowtime().timestampsFromSource().watermarksFromSource())
val props = new DescriptorProperties()
- desc1.addProperties(props)
+ props.putProperties(desc1.toProperties)
val inputSchema = TableSchema.builder()
.field("csvField", Types.STRING)
@@ -69,14 +69,6 @@ class SchemaValidatorTest {
assertEquals(
expectedMapping,
SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema.toRowType)))
-
- // test field format
- val formatSchema = SchemaValidator.deriveFormatFields(props)
- val expectedFormatSchema = TableSchema.builder()
- .field("csvField", Types.STRING) // aliased
- .field("abcField", Types.STRING)
- .build()
- assertEquals(expectedFormatSchema, formatSchema)
}
@Test(expected = classOf[TableException])
@@ -88,7 +80,7 @@ class SchemaValidatorTest {
.field("r", Types.SQL_TIMESTAMP).rowtime(
Rowtime().timestampsFromSource().watermarksFromSource())
val props = new DescriptorProperties()
- desc1.addProperties(props)
+ props.putProperties(desc1.toProperties)
SchemaValidator.deriveTableSinkSchema(props)
}
@@ -102,7 +94,7 @@ class SchemaValidatorTest {
.field("r", Types.SQL_TIMESTAMP).rowtime(
Rowtime().timestampsFromField("myTime").watermarksFromSource())
val props = new DescriptorProperties()
- desc1.addProperties(props)
+ props.putProperties(desc1.toProperties)
val expectedTableSinkSchema = TableSchema.builder()
.field("csvField", Types.STRING) // aliased
@@ -122,7 +114,7 @@ class SchemaValidatorTest {
.field("r", Types.SQL_TIMESTAMP).rowtime(
Rowtime().timestampsFromField("myTime").watermarksFromSource())
val props = new DescriptorProperties()
- desc1.addProperties(props)
+ props.putProperties(desc1.toProperties)
val inputSchema = TableSchema.builder()
.field("csvField", Types.STRING)
@@ -150,15 +142,6 @@ class SchemaValidatorTest {
assertEquals(
expectedMapping,
SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema.toRowType)))
-
- // test field format
- val formatSchema = SchemaValidator.deriveFormatFields(props)
- val expectedFormatSchema = TableSchema.builder()
- .field("csvField", Types.STRING) // aliased
- .field("abcField", Types.STRING)
- .field("myTime", Types.SQL_TIMESTAMP)
- .build()
- assertEquals(expectedFormatSchema, formatSchema)
}
@Test
@@ -171,7 +154,7 @@ class SchemaValidatorTest {
Rowtime().timestampsFromExtractor(new CustomExtractor("f3"))
.watermarksPeriodicBounded(1000L))
val properties = new DescriptorProperties()
- descriptor.addProperties(properties)
+ properties.putProperties(descriptor.toProperties)
val rowtime = SchemaValidator.deriveRowtimeAttributes(properties).get(0)
assertEquals("rt", rowtime.getAttributeName)
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TableDescriptorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TableDescriptorTest.scala
index 00e3a21..dcc1ab9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TableDescriptorTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TableDescriptorTest.scala
@@ -130,9 +130,6 @@ class TableDescriptorTest extends TableTestBase {
expectedCommonProperties
}
- val actualProperties = new DescriptorProperties(true)
- descriptor.addProperties(actualProperties)
-
- assertEquals(expectedProperties.toMap.asJava, actualProperties.asMap)
+ assertEquals(expectedProperties.toMap.asJava, descriptor.toProperties)
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TestTableDescriptor.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TestTableDescriptor.scala
new file mode 100644
index 0000000..cb84953
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/TestTableDescriptor.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors
+
+import java.util
+
+import org.apache.flink.table.descriptors.StreamTableDescriptorValidator.{UPDATE_MODE, UPDATE_MODE_VALUE_APPEND, UPDATE_MODE_VALUE_RETRACT, UPDATE_MODE_VALUE_UPSERT}
+
+class TestTableDescriptor(connector: ConnectorDescriptor)
+ extends TableDescriptor
+ with SchematicDescriptor[TestTableDescriptor]
+ with StreamableDescriptor[TestTableDescriptor] {
+
+ private var formatDescriptor: Option[FormatDescriptor] = None
+ private var schemaDescriptor: Option[Schema] = None
+ private var updateMode: Option[String] = None
+
+ override def toProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
+ properties.putProperties(connector.toProperties)
+ formatDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ schemaDescriptor.foreach(d => properties.putProperties(d.toProperties))
+ updateMode.foreach(mode => properties.putString(UPDATE_MODE, mode))
+
+ properties.asMap()
+ }
+
+ override def withFormat(format: FormatDescriptor): TestTableDescriptor = {
+ this.formatDescriptor = Some(format)
+ this
+ }
+
+ override def withSchema(schema: Schema): TestTableDescriptor = {
+ this.schemaDescriptor = Some(schema)
+ this
+ }
+
+ override def inAppendMode(): TestTableDescriptor = {
+ updateMode = Some(UPDATE_MODE_VALUE_APPEND)
+ this
+ }
+
+ override def inRetractMode(): TestTableDescriptor = {
+ updateMode = Some(UPDATE_MODE_VALUE_RETRACT)
+ this
+ }
+
+ override def inUpsertMode(): TestTableDescriptor = {
+ updateMode = Some(UPDATE_MODE_VALUE_UPSERT)
+ this
+ }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormat.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormat.scala
index f00b6a1..3a1de49 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormat.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormat.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.factories.utils
+import java.util
+
import org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA
import org.apache.flink.table.descriptors.{DescriptorProperties, FormatDescriptor}
import org.apache.flink.table.factories.TableFormatFactoryServiceTest._
@@ -27,8 +29,12 @@ import org.apache.flink.table.factories.TableFormatFactoryServiceTest._
*/
class TestTableFormat extends FormatDescriptor(TEST_FORMAT_TYPE, 1) {
- override protected def addFormatProperties(properties: DescriptorProperties): Unit = {
+ override protected def toFormatProperties: util.Map[String, String] = {
+ val properties = new DescriptorProperties()
+
properties.putString(COMMON_PATH, "/path/to/sth")
properties.putString(FORMAT_DERIVE_SCHEMA, "true")
+
+ properties.asMap()
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormatFactory.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormatFactory.scala
index 39c268e..9344f23 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormatFactory.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/factories/utils/TestTableFormatFactory.scala
@@ -21,8 +21,8 @@ package org.apache.flink.table.factories.utils
import java.util
import org.apache.flink.api.common.serialization.{DeserializationSchema, SerializationSchema}
-import org.apache.flink.table.descriptors.{DescriptorProperties, FormatDescriptorValidator, SchemaValidator}
-import org.apache.flink.table.factories.{DeserializationSchemaFactory, SerializationSchemaFactory, TableFormatFactoryServiceTest}
+import org.apache.flink.table.factories.TableFormatFactoryBase.deriveSchema
+import org.apache.flink.table.factories.{DeserializationSchemaFactory, SerializationSchemaFactory, TableFormatFactoryBase, TableFormatFactoryServiceTest}
import org.apache.flink.types.Row
/**
@@ -32,26 +32,14 @@ import org.apache.flink.types.Row
* This format does not support SPECIAL_PATH but supports schema derivation.
*/
class TestTableFormatFactory
- extends DeserializationSchemaFactory[Row]
+ extends TableFormatFactoryBase[Row](TableFormatFactoryServiceTest.TEST_FORMAT_TYPE, 1, true)
+ with DeserializationSchemaFactory[Row]
with SerializationSchemaFactory[Row] {
- override def requiredContext(): util.Map[String, String] = {
- val context = new util.HashMap[String, String]()
- context.put(
- FormatDescriptorValidator.FORMAT_TYPE,
- TableFormatFactoryServiceTest.TEST_FORMAT_TYPE)
- context.put(FormatDescriptorValidator.FORMAT_PROPERTY_VERSION, "1")
- context
- }
-
- override def supportsSchemaDerivation(): Boolean = true
-
- override def supportedProperties(): util.List[String] = {
+ override def supportedFormatProperties(): util.List[String] = {
val properties = new util.ArrayList[String]()
properties.add(TableFormatFactoryServiceTest.UNIQUE_PROPERTY)
properties.add(TableFormatFactoryServiceTest.COMMON_PATH)
- properties.add(FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA)
- properties.addAll(SchemaValidator.getSchemaDerivationKeys)
properties
}
@@ -59,19 +47,13 @@ class TestTableFormatFactory
properties: util.Map[String, String])
: DeserializationSchema[Row] = {
- val props = new DescriptorProperties(true)
- props.putProperties(properties)
- val schema = SchemaValidator.deriveFormatFields(props)
- new TestDeserializationSchema(schema.toRowType)
+ new TestDeserializationSchema(deriveSchema(properties).toRowType)
}
override def createSerializationSchema(
properties: util.Map[String, String])
: SerializationSchema[Row] = {
- val props = new DescriptorProperties(true)
- props.putProperties(properties)
- val schema = SchemaValidator.deriveFormatFields(props)
- new TestSerializationSchema(schema.toRowType)
+ new TestSerializationSchema(deriveSchema(properties).toRowType)
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
index de0b392..c3da65f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/stream/sql/SqlITCase.scala
@@ -25,16 +25,15 @@ import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.watermark.Watermark
-import org.apache.flink.table.api.{TableEnvironment, Types}
import org.apache.flink.table.api.scala._
-import org.apache.flink.table.descriptors.{DescriptorProperties, Rowtime, Schema}
-import org.apache.flink.table.expressions.utils.SplitUDF
+import org.apache.flink.table.api.{TableEnvironment, Types}
+import org.apache.flink.table.descriptors.{Rowtime, Schema}
import org.apache.flink.table.expressions.utils.Func15
import org.apache.flink.table.runtime.stream.sql.SqlITCase.TimestampAndWatermarkWithOffset
import org.apache.flink.table.runtime.utils.TimeTestUtil.EventTimeSourceFunction
import org.apache.flink.table.runtime.utils.{JavaUserDefinedTableFunctions, StreamITCase, StreamTestData, StreamingWithStateTestBase}
-import org.apache.flink.types.Row
import org.apache.flink.table.utils.{InMemoryTableFactory, MemoryTableSourceSinkUtil}
+import org.apache.flink.types.Row
import org.junit.Assert._
import org.junit._
@@ -727,8 +726,7 @@ class SqlITCase extends StreamingWithStateTestBase {
.field("t", Types.SQL_TIMESTAMP)
.rowtime(Rowtime().timestampsFromField("t").watermarksPeriodicAscending())
.field("proctime", Types.SQL_TIMESTAMP).proctime()
- val props = new DescriptorProperties()
- desc.addProperties(props)
+ val properties = desc.toProperties
val t = StreamTestData.getSmall3TupleDataStream(env)
.assignAscendingTimestamps(x => x._2)
@@ -736,9 +734,9 @@ class SqlITCase extends StreamingWithStateTestBase {
tEnv.registerTable("sourceTable", t)
tEnv.registerTableSource("targetTable",
- new InMemoryTableFactory(3).createStreamTableSource(props.asMap))
+ new InMemoryTableFactory(3).createStreamTableSource(properties))
tEnv.registerTableSink("targetTable",
- new InMemoryTableFactory(3).createStreamTableSink(props.asMap))
+ new InMemoryTableFactory(3).createStreamTableSink(properties))
tEnv.sqlUpdate("INSERT INTO targetTable SELECT a, b, c, rowtime FROM sourceTable")
tEnv.sqlQuery("SELECT a, e, f, t from targetTable")
[flink] 04/07: [FLINK-10687] [table] Move TypeStringUtils to
flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 026f43a2d2ec1883757e9949879450c6350be0e1
Author: Timo Walther <tw...@apache.org>
AuthorDate: Fri Oct 26 15:57:03 2018 +0200
[FLINK-10687] [table] Move TypeStringUtils to flink-table-common
---
.../org/apache/flink/table/descriptors/Json.java | 2 +-
.../apache/flink/table/utils/EncodingUtils.java | 215 +++++++++
.../apache/flink/table/utils/TypeStringUtils.java | 495 +++++++++++++++++++++
.../flink/table/utils/TypeStringUtilsTest.java | 179 ++++++++
.../table/descriptors/DescriptorProperties.scala | 6 +-
.../flink/table/descriptors/LiteralValue.scala | 2 +-
.../flink/table/typeutils/TypeStringUtils.scala | 299 -------------
.../table/typeutils/TypeStringUtilsTest.scala | 124 ------
8 files changed, 894 insertions(+), 428 deletions(-)
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
index 026f49e..8e5b6f3 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
@@ -19,7 +19,7 @@
package org.apache.flink.table.descriptors;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.table.typeutils.TypeStringUtils;
+import org.apache.flink.table.utils.TypeStringUtils;
import org.apache.flink.types.Row;
import org.apache.flink.util.Preconditions;
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
new file mode 100644
index 0000000..41fa58e
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/EncodingUtils.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.utils;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+/**
+ * General utilities for string-encoding. This class is used to avoid additional dependencies
+ * to other projects.
+ */
+@Internal
+public abstract class EncodingUtils {
+
+ private static final Base64.Encoder BASE64_ENCODER = java.util.Base64.getUrlEncoder().withoutPadding();
+
+ private static final Base64.Decoder BASE64_DECODER = java.util.Base64.getUrlDecoder();
+
+ private EncodingUtils() {
+ // do not instantiate
+ }
+
+ public static String encodeObjectToString(Serializable obj) {
+ try {
+ final byte[] bytes = InstantiationUtil.serializeObject(obj);
+ return new String(BASE64_ENCODER.encode(bytes), StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ throw new ValidationException(
+ "Unable to serialize object '" + obj.toString() + "' of class '" + obj.getClass().getName() + "'.");
+ }
+ }
+
+ public static <T extends Serializable> T decodeStringToObject(String base64String, Class<T> baseClass) {
+ return decodeStringToObject(base64String, baseClass, Thread.currentThread().getContextClassLoader());
+ }
+
+ public static <T extends Serializable> T decodeStringToObject(String base64String, Class<T> baseClass, ClassLoader classLoader) {
+ try {
+ final byte[] bytes = BASE64_DECODER.decode(base64String.getBytes(StandardCharsets.UTF_8));
+ final T instance = InstantiationUtil.deserializeObject(bytes, classLoader);
+ if (instance != null && !baseClass.isAssignableFrom(instance.getClass())) {
+ throw new ValidationException(
+ "Object '" + instance + "' does not match expected base class '" + baseClass +
+ "' but is of class '" + instance.getClass() + "'.");
+ }
+ return instance;
+ } catch (Exception e) {
+ throw new ValidationException(
+ "Unable to deserialize string '" + base64String + "' of base class '" + baseClass.getName() + "'.");
+ }
+ }
+
+ public static Class<?> loadClass(String qualifiedName, ClassLoader classLoader) {
+ try {
+ return Class.forName(qualifiedName, true, classLoader);
+ } catch (Exception e) {
+ throw new ValidationException("Class '" + qualifiedName + "' could not be loaded. " +
+ "Please note that inner classes must be globally accessible and declared static.", e);
+ }
+ }
+
+ public static Class<?> loadClass(String qualifiedName) {
+ return loadClass(qualifiedName, Thread.currentThread().getContextClassLoader());
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Java String Escaping
+ //
+ // copied from o.a.commons.lang.StringEscapeUtils (commons-lang:commons-lang:2.4)
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Escapes the characters in a <code>String</code> using Java String rules.
+ *
+ * <p>Deals correctly with quotes and control-chars (tab, backslash, cr, ff, etc.)
+ *
+ * <p>So a tab becomes the characters <code>'\\'</code> and <code>'t'</code>.
+ *
+ * <p>The only difference between Java strings and JavaScript strings
+ * is that in JavaScript, a single quote must be escaped.
+ *
+ * <p>Example:
+ * <pre>
+ * input string: He didn't say, "Stop!"
+ * output string: He didn't say, \"Stop!\"
+ * </pre>
+ * </p>
+ *
+ * @param str String to escape values in, may be null
+ * @return String with escaped values, <code>null</code> if null string input
+ */
+ public static String escapeJava(String str) {
+ return escapeJavaStyleString(str, false);
+ }
+
+ private static String escapeJavaStyleString(String str, boolean escapeSingleQuotes) {
+ if (str == null) {
+ return null;
+ }
+ try {
+ StringWriter writer = new StringWriter(str.length() * 2);
+ escapeJavaStyleString(writer, str, escapeSingleQuotes);
+ return writer.toString();
+ } catch (IOException ioe) {
+ // this should never ever happen while writing to a StringWriter
+ ioe.printStackTrace();
+ return null;
+ }
+ }
+
+ private static void escapeJavaStyleString(Writer out, String str, boolean escapeSingleQuote) throws IOException {
+ if (out == null) {
+ throw new IllegalArgumentException("The Writer must not be null");
+ }
+ if (str == null) {
+ return;
+ }
+ int sz;
+ sz = str.length();
+ for (int i = 0; i < sz; i++) {
+ char ch = str.charAt(i);
+
+ // handle unicode
+ if (ch > 0xfff) {
+ out.write("\\u" + hex(ch));
+ } else if (ch > 0xff) {
+ out.write("\\u0" + hex(ch));
+ } else if (ch > 0x7f) {
+ out.write("\\u00" + hex(ch));
+ } else if (ch < 32) {
+ switch (ch) {
+ case '\b':
+ out.write('\\');
+ out.write('b');
+ break;
+ case '\n':
+ out.write('\\');
+ out.write('n');
+ break;
+ case '\t':
+ out.write('\\');
+ out.write('t');
+ break;
+ case '\f':
+ out.write('\\');
+ out.write('f');
+ break;
+ case '\r':
+ out.write('\\');
+ out.write('r');
+ break;
+ default:
+ if (ch > 0xf) {
+ out.write("\\u00" + hex(ch));
+ } else {
+ out.write("\\u000" + hex(ch));
+ }
+ break;
+ }
+ } else {
+ switch (ch) {
+ case '\'':
+ if (escapeSingleQuote) {
+ out.write('\\');
+ }
+ out.write('\'');
+ break;
+ case '"':
+ out.write('\\');
+ out.write('"');
+ break;
+ case '\\':
+ out.write('\\');
+ out.write('\\');
+ break;
+ case '/':
+ out.write('\\');
+ out.write('/');
+ break;
+ default:
+ out.write(ch);
+ break;
+ }
+ }
+ }
+ }
+
+ private static String hex(char ch) {
+ return Integer.toHexString(ch).toUpperCase();
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/TypeStringUtils.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/TypeStringUtils.java
new file mode 100644
index 0000000..faf26d0
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/utils/TypeStringUtils.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.utils;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.functions.InvalidTypesException;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.GenericTypeInfo;
+import org.apache.flink.api.java.typeutils.MapTypeInfo;
+import org.apache.flink.api.java.typeutils.MultisetTypeInfo;
+import org.apache.flink.api.java.typeutils.ObjectArrayTypeInfo;
+import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.ValidationException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Utilities to convert {@link TypeInformation} into a string representation and back.
+ */
+@PublicEvolving
+public class TypeStringUtils {
+
+ private static final String VARCHAR = "VARCHAR";
+ private static final String STRING = "STRING";
+ private static final String BOOLEAN = "BOOLEAN";
+ private static final String BYTE = "BYTE";
+ private static final String TINYINT = "TINYINT";
+ private static final String SHORT = "SHORT";
+ private static final String SMALLINT = "SMALLINT";
+ private static final String INT = "INT";
+ private static final String LONG = "LONG";
+ private static final String BIGINT = "BIGINT";
+ private static final String FLOAT = "FLOAT";
+ private static final String DOUBLE = "DOUBLE";
+ private static final String DECIMAL = "DECIMAL";
+ private static final String SQL_DATE = "SQL_DATE";
+ private static final String DATE = "DATE";
+ private static final String SQL_TIME = "SQL_TIME";
+ private static final String TIME = "TIME";
+ private static final String SQL_TIMESTAMP = "SQL_TIMESTAMP";
+ private static final String TIMESTAMP = "TIMESTAMP";
+ private static final String ROW = "ROW";
+ private static final String ANY = "ANY";
+ private static final String POJO = "POJO";
+ private static final String MAP = "MAP";
+ private static final String MULTISET = "MULTISET";
+ private static final String PRIMITIVE_ARRAY = "PRIMITIVE_ARRAY";
+ private static final String OBJECT_ARRAY = "OBJECT_ARRAY";
+
+ public static String writeTypeInfo(TypeInformation<?> typeInfo) {
+ if (typeInfo.equals(Types.STRING)) {
+ return VARCHAR;
+ } else if (typeInfo.equals(Types.BOOLEAN)) {
+ return BOOLEAN;
+ } else if (typeInfo.equals(Types.BYTE)) {
+ return TINYINT;
+ } else if (typeInfo.equals(Types.SHORT)) {
+ return SMALLINT;
+ } else if (typeInfo.equals(Types.INT)) {
+ return INT;
+ } else if (typeInfo.equals(Types.LONG)) {
+ return BIGINT;
+ } else if (typeInfo.equals(Types.FLOAT)) {
+ return FLOAT;
+ } else if (typeInfo.equals(Types.DOUBLE)) {
+ return DOUBLE;
+ } else if (typeInfo.equals(Types.BIG_DEC)) {
+ return DECIMAL;
+ } else if (typeInfo.equals(Types.SQL_DATE)) {
+ return DATE;
+ } else if (typeInfo.equals(Types.SQL_TIME)) {
+ return TIME;
+ } else if (typeInfo.equals(Types.SQL_TIMESTAMP)) {
+ return TIMESTAMP;
+ } else if (typeInfo instanceof RowTypeInfo) {
+ final RowTypeInfo rt = (RowTypeInfo) typeInfo;
+ final String[] fieldNames = rt.getFieldNames();
+ final TypeInformation<?>[] fieldTypes = rt.getFieldTypes();
+
+ final StringBuilder result = new StringBuilder();
+ result.append(ROW);
+ result.append('<');
+ for (int i = 0; i < fieldNames.length; i++) {
+ // escape field name if it contains delimiters
+ if (containsDelimiter(fieldNames[i])) {
+ result.append('`');
+ result.append(fieldNames[i].replace("`", "``"));
+ result.append('`');
+ } else {
+ result.append(fieldNames[i]);
+ }
+ result.append(' ');
+ result.append(writeTypeInfo(fieldTypes[i]));
+ if (i < fieldNames.length - 1) {
+ result.append(", ");
+ }
+ }
+ result.append('>');
+ return result.toString();
+ } else if (typeInfo instanceof GenericTypeInfo) {
+ return ANY + '<' + typeInfo.getTypeClass().getName() + '>';
+ } else if (typeInfo instanceof PojoTypeInfo) {
+ // we only support very simple POJOs that only contain extracted fields
+ // (not manually specified)
+ TypeInformation<?> extractedPojo;
+ try {
+ extractedPojo = TypeExtractor.createTypeInfo(typeInfo.getTypeClass());
+ } catch (InvalidTypesException e) {
+ extractedPojo = null;
+ }
+ if (extractedPojo == null || !typeInfo.equals(extractedPojo)) {
+ throw new TableException(
+ "A string representation for custom POJO types is not supported yet.");
+ }
+ return POJO + '<' + typeInfo.getTypeClass().getName() + '>';
+ } else if (typeInfo instanceof PrimitiveArrayTypeInfo) {
+ final PrimitiveArrayTypeInfo arrayTypeInfo = (PrimitiveArrayTypeInfo) typeInfo;
+ return PRIMITIVE_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentType()) + '>';
+ } else if (typeInfo instanceof ObjectArrayTypeInfo) {
+ final ObjectArrayTypeInfo arrayTypeInfo = (ObjectArrayTypeInfo) typeInfo;
+ return OBJECT_ARRAY + '<' + writeTypeInfo(arrayTypeInfo.getComponentInfo()) + '>';
+ } else if (typeInfo instanceof MultisetTypeInfo) {
+ final MultisetTypeInfo multisetTypeInfo = (MultisetTypeInfo) typeInfo;
+ return MULTISET + '<' + writeTypeInfo(multisetTypeInfo.getElementTypeInfo()) + '>';
+ } else if (typeInfo instanceof MapTypeInfo) {
+ final MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
+ final String keyTypeInfo = writeTypeInfo(mapTypeInfo.getKeyTypeInfo());
+ final String valueTypeInfo = writeTypeInfo(mapTypeInfo.getValueTypeInfo());
+ return MAP + '<' + keyTypeInfo + ", " + valueTypeInfo + '>';
+ } else {
+ return ANY + '<' + typeInfo.getTypeClass().getName() + ", " +
+ EncodingUtils.encodeObjectToString(typeInfo) + '>';
+ }
+ }
+
+ public static TypeInformation<?> readTypeInfo(String typeString) {
+ final List<Token> tokens = tokenize(typeString);
+ final TokenConverter converter = new TokenConverter(typeString, tokens);
+ return converter.convert();
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private static boolean containsDelimiter(String string) {
+ final char[] charArray = string.toCharArray();
+ for (char c : charArray) {
+ if (isDelimiter(c)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isDelimiter(char character) {
+ return Character.isWhitespace(character) || character == ',' ||
+ character == '<' || character == '>' || character == '(' || character == ')';
+ }
+
+ private static List<Token> tokenize(String typeString) {
+ final char[] chars = typeString.toCharArray();
+
+ final List<Token> tokens = new ArrayList<>();
+ for (int cursor = 0; cursor < chars.length; cursor++) {
+ char curChar = chars[cursor];
+ if (curChar == '<' || curChar == '(') {
+ tokens.add(new Token(TokenType.BEGIN, Character.toString(curChar), cursor));
+ } else if (curChar == '>' || curChar == ')') {
+ tokens.add(new Token(TokenType.END, Character.toString(curChar), cursor));
+ } else if (curChar == ',') {
+ tokens.add(new Token(TokenType.SEPARATOR, Character.toString(curChar), cursor));
+ } else if (!Character.isWhitespace(curChar)) {
+ // parse literal
+ final StringBuilder literal = new StringBuilder();
+ boolean isEscaped = false;
+ while (cursor < chars.length && (!isDelimiter(chars[cursor]) || isEscaped)) {
+ curChar = chars[cursor++];
+ if (!isEscaped && curChar == '`') {
+ isEscaped = true;
+ } else if (isEscaped && curChar == '`' && cursor < chars.length && chars[cursor] == '`') {
+ // escaped backtick using "`Hello `` World`"
+ cursor++;
+ literal.append(curChar);
+ } else if (isEscaped && curChar == '`') {
+ break;
+ } else {
+ literal.append(curChar);
+ }
+ }
+ cursor -= 1;
+ tokens.add(new Token(TokenType.LITERAL, literal.toString(), cursor));
+ }
+ }
+
+ return tokens;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private enum TokenType {
+ LITERAL, BEGIN, END, SEPARATOR
+ }
+
+ private static class Token {
+ public final TokenType type;
+ public final String literal;
+ public final int cursorPosition;
+
+ public Token(TokenType type, String literal, int cursorPosition) {
+ this.type = type;
+ this.literal = literal;
+ this.cursorPosition = cursorPosition;
+ }
+ }
+
+ private static class TokenConverter {
+
+ private String inputString;
+ private List<Token> tokens;
+ private int lastValidToken;
+ private int currentToken;
+
+ public TokenConverter(String inputString, List<Token> tokens) {
+ this.inputString = inputString;
+ this.tokens = tokens;
+ this.lastValidToken = -1;
+ this.currentToken = -1;
+ }
+
+ public TypeInformation<?> convert() {
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> typeInfo = convertType();
+ if (hasRemainingTokens()) {
+ nextToken();
+ throw parsingError("Unexpected token: " + token().literal);
+ }
+ return typeInfo;
+ }
+
+ private TypeInformation<?> convertType() {
+ final TypeInformation<?> typeInfo;
+ switch (token().literal) {
+ case VARCHAR:
+ case STRING:
+ return Types.STRING;
+ case BOOLEAN:
+ return Types.BOOLEAN;
+ case TINYINT:
+ case BYTE:
+ return Types.BYTE;
+ case SMALLINT:
+ case SHORT:
+ return Types.SHORT;
+ case INT:
+ return Types.INT;
+ case BIGINT:
+ case LONG:
+ return Types.LONG;
+ case FLOAT:
+ return Types.FLOAT;
+ case DOUBLE:
+ return Types.DOUBLE;
+ case DECIMAL:
+ return Types.BIG_DEC;
+ case DATE:
+ case SQL_DATE:
+ return Types.SQL_DATE;
+ case TIMESTAMP:
+ case SQL_TIMESTAMP:
+ return Types.SQL_TIMESTAMP;
+ case TIME:
+ case SQL_TIME:
+ return Types.SQL_TIME;
+ case ROW:
+ return convertRow();
+ case ANY:
+ return convertAny();
+ case POJO:
+ return convertPojo();
+ case MAP:
+ return convertMap();
+ case MULTISET:
+ return convertMultiset();
+ case PRIMITIVE_ARRAY:
+ return convertPrimitiveArray();
+ case OBJECT_ARRAY:
+ return convertObjectArray();
+ default:
+ throw parsingError("Unsupported type: " + token().literal);
+ }
+ }
+
+ private TypeInformation<?> convertRow() {
+ nextToken(TokenType.BEGIN);
+
+ // check if ROW<INT, INT> or ROW<name INT, other INT>
+ if (isNextToken(2, TokenType.LITERAL)) {
+ // named row
+ final List<String> names = new ArrayList<>();
+ final List<TypeInformation<?>> types = new ArrayList<>();
+ while (hasRemainingTokens()) {
+ nextToken(TokenType.LITERAL);
+ names.add(token().literal);
+
+ nextToken(TokenType.LITERAL);
+ types.add(convertType());
+
+ if (isNextToken(1, TokenType.END)) {
+ break;
+ }
+ nextToken(TokenType.SEPARATOR);
+ }
+
+ nextToken(TokenType.END);
+
+ return Types.ROW_NAMED(
+ names.toArray(new String[0]),
+ types.toArray(new TypeInformation<?>[0]));
+ } else {
+ // unnamed row
+ final List<TypeInformation<?>> types = new ArrayList<>();
+ while (hasRemainingTokens()) {
+ nextToken(TokenType.LITERAL);
+ types.add(convertType());
+
+ if (isNextToken(1, TokenType.END)) {
+ break;
+ }
+ nextToken(TokenType.SEPARATOR);
+ }
+
+ nextToken(TokenType.END);
+
+ return Types.ROW(types.toArray(new TypeInformation<?>[0]));
+ }
+ }
+
+ private TypeInformation<?> convertAny() {
+ nextToken(TokenType.BEGIN);
+
+ // check if ANY(class) or ANY(class, serialized)
+ if (isNextToken(2, TokenType.SEPARATOR)) {
+ // any type information
+ nextToken(TokenType.LITERAL);
+ final String className = token().literal;
+
+ nextToken(TokenType.SEPARATOR);
+
+ nextToken(TokenType.LITERAL);
+ final String serialized = token().literal;
+
+ nextToken(TokenType.END);
+
+ final Class<?> clazz = EncodingUtils.loadClass(className);
+ final TypeInformation<?> typeInfo = EncodingUtils.decodeStringToObject(serialized, TypeInformation.class);
+ if (!clazz.equals(typeInfo.getTypeClass())) {
+ throw new ValidationException("Class '" + clazz + "' does no correspond to serialized data.");
+ }
+ return typeInfo;
+ } else {
+ // generic type information
+ nextToken(TokenType.LITERAL);
+ final String className = token().literal;
+
+ nextToken(TokenType.END);
+
+ final Class<?> clazz = EncodingUtils.loadClass(className);
+ return Types.GENERIC(clazz);
+ }
+ }
+
+ private TypeInformation<?> convertPojo() {
+ nextToken(TokenType.BEGIN);
+
+ nextToken(TokenType.LITERAL);
+ final String className = token().literal;
+
+ nextToken(TokenType.END);
+
+ final Class<?> clazz = EncodingUtils.loadClass(className);
+ return Types.POJO(clazz);
+ }
+
+ private TypeInformation<?> convertMap() {
+ nextToken(TokenType.BEGIN);
+
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> keyTypeInfo = convertType();
+
+ nextToken(TokenType.SEPARATOR);
+
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> valueTypeInfo = convertType();
+
+ nextToken(TokenType.END);
+
+ return Types.MAP(keyTypeInfo, valueTypeInfo);
+ }
+
+ private TypeInformation<?> convertMultiset() {
+ nextToken(TokenType.BEGIN);
+
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> elementTypeInfo = convertType();
+
+ nextToken(TokenType.END);
+
+ return new MultisetTypeInfo<>(elementTypeInfo);
+ }
+
+ private TypeInformation<?> convertPrimitiveArray() {
+ nextToken(TokenType.BEGIN);
+
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> elementTypeInfo = convertType();
+
+ nextToken(TokenType.END);
+
+ return Types.PRIMITIVE_ARRAY(elementTypeInfo);
+ }
+
+ private TypeInformation<?> convertObjectArray() {
+ nextToken(TokenType.BEGIN);
+
+ nextToken(TokenType.LITERAL);
+ final TypeInformation<?> elementTypeInfo = convertType();
+
+ nextToken(TokenType.END);
+
+ return Types.OBJECT_ARRAY(elementTypeInfo);
+ }
+
+ private void nextToken(TokenType type) {
+ nextToken();
+ final Token nextToken = tokens.get(currentToken);
+ if (nextToken.type != type) {
+ throw parsingError(type.name() + " expected but was " + nextToken.type + '.');
+ }
+ }
+
+ private void nextToken() {
+ this.currentToken++;
+ if (currentToken >= tokens.size()) {
+ throw parsingError("Unexpected end.");
+ }
+ lastValidToken = this.currentToken - 1;
+ }
+
+ private boolean isNextToken(int lookAhead, TokenType type) {
+ return currentToken + lookAhead < tokens.size() &&
+ tokens.get(currentToken + lookAhead).type == type;
+ }
+
+ private int lastCursor() {
+ if (lastValidToken < 0) {
+ return 0;
+ }
+ return tokens.get(lastValidToken).cursorPosition + 1;
+ }
+
+ private ValidationException parsingError(String cause) {
+ throw new ValidationException("Could not parse type information at position " + lastCursor() + ": " + cause + "\n" +
+ "Input type string: " + inputString);
+ }
+
+ private Token token() {
+ return tokens.get(currentToken);
+ }
+
+ private boolean hasRemainingTokens() {
+ return currentToken + 1 < tokens.size();
+ }
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/utils/TypeStringUtilsTest.java b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/utils/TypeStringUtilsTest.java
new file mode 100644
index 0000000..746a8a1
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/test/java/org/apache/flink/table/utils/TypeStringUtilsTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.utils;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.MultisetTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.table.api.ValidationException;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link TypeStringUtils}.
+ */
+public class TypeStringUtilsTest {
+
+ @Test
+ public void testPrimitiveTypes() {
+ testReadAndWrite("VARCHAR", Types.STRING);
+ testReadAndWrite("BOOLEAN", Types.BOOLEAN);
+ testReadAndWrite("TINYINT", Types.BYTE);
+ testReadAndWrite("SMALLINT", Types.SHORT);
+ testReadAndWrite("INT", Types.INT);
+ testReadAndWrite("BIGINT", Types.LONG);
+ testReadAndWrite("FLOAT", Types.FLOAT);
+ testReadAndWrite("DOUBLE", Types.DOUBLE);
+ testReadAndWrite("DECIMAL", Types.BIG_DEC);
+ testReadAndWrite("DATE", Types.SQL_DATE);
+ testReadAndWrite("TIME", Types.SQL_TIME);
+ testReadAndWrite("TIMESTAMP", Types.SQL_TIMESTAMP);
+
+ // unsupported type information
+ testReadAndWrite(
+ "ANY<java.lang.Void, " +
+ "rO0ABXNyADJvcmcuYXBhY2hlLmZsaW5rLmFwaS5jb21tb24udHlwZWluZm8uQmFzaWNUeXBlSW5mb_oE8IKl" +
+ "ad0GAgAETAAFY2xhenp0ABFMamF2YS9sYW5nL0NsYXNzO0wAD2NvbXBhcmF0b3JDbGFzc3EAfgABWwAXcG9z" +
+ "c2libGVDYXN0VGFyZ2V0VHlwZXN0ABJbTGphdmEvbGFuZy9DbGFzcztMAApzZXJpYWxpemVydAA2TG9yZy9h" +
+ "cGFjaGUvZmxpbmsvYXBpL2NvbW1vbi90eXBldXRpbHMvVHlwZVNlcmlhbGl6ZXI7eHIANG9yZy5hcGFjaGUu" +
+ "ZmxpbmsuYXBpLmNvbW1vbi50eXBlaW5mby5UeXBlSW5mb3JtYXRpb26UjchIurN66wIAAHhwdnIADmphdmEu" +
+ "bGFuZy5Wb2lkAAAAAAAAAAAAAAB4cHB1cgASW0xqYXZhLmxhbmcuQ2xhc3M7qxbXrsvNWpkCAAB4cAAAAABz" +
+ "cgA5b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1dGlscy5iYXNlLlZvaWRTZXJpYWxpemVyAAAA" +
+ "AAAAAAECAAB4cgBCb3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1dGlscy5iYXNlLlR5cGVTZXJp" +
+ "YWxpemVyU2luZ2xldG9ueamHqscud0UCAAB4cgA0b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1" +
+ "dGlscy5UeXBlU2VyaWFsaXplcgAAAAAAAAABAgAAeHA>",
+ Types.VOID);
+ }
+
+ @Test
+ public void testWriteComplexTypes() {
+ testReadAndWrite(
+ "ROW<f0 DECIMAL, f1 TINYINT>",
+ Types.ROW(Types.BIG_DEC, Types.BYTE));
+
+ testReadAndWrite(
+ "ROW<hello DECIMAL, world TINYINT>",
+ Types.ROW_NAMED(
+ new String[]{"hello", "world"},
+ Types.BIG_DEC, Types.BYTE));
+
+ testReadAndWrite(
+ "POJO<org.apache.flink.table.utils.TypeStringUtilsTest$TestPojo>",
+ TypeExtractor.createTypeInfo(TestPojo.class));
+
+ testReadAndWrite(
+ "ANY<org.apache.flink.table.utils.TypeStringUtilsTest$TestNoPojo>",
+ TypeExtractor.createTypeInfo(TestNoPojo.class));
+
+ testReadAndWrite(
+ "MAP<VARCHAR, ROW<f0 DECIMAL, f1 TINYINT>>",
+ Types.MAP(Types.STRING, Types.ROW(Types.BIG_DEC, Types.BYTE)));
+
+ testReadAndWrite(
+ "MULTISET<ROW<f0 DECIMAL, f1 TINYINT>>",
+ new MultisetTypeInfo<>(Types.ROW(Types.BIG_DEC, Types.BYTE)));
+
+ testReadAndWrite(
+ "PRIMITIVE_ARRAY<TINYINT>",
+ Types.PRIMITIVE_ARRAY(Types.BYTE));
+
+ testReadAndWrite(
+ "OBJECT_ARRAY<POJO<org.apache.flink.table.utils.TypeStringUtilsTest$TestPojo>>",
+ Types.OBJECT_ARRAY(TypeExtractor.createTypeInfo(TestPojo.class)));
+
+ // test escaping
+ assertEquals(
+ Types.ROW_NAMED(
+ new String[] {"he \nllo", "world"},
+ Types.BIG_DEC, Types.BYTE),
+ TypeStringUtils.readTypeInfo("ROW<`he \nllo` DECIMAL, world TINYINT>"));
+
+ assertEquals(
+ Types.ROW_NAMED(
+ new String[] {"he`llo", "world"},
+ Types.BIG_DEC, Types.BYTE),
+ TypeStringUtils.readTypeInfo("ROW<`he``llo` DECIMAL, world TINYINT>"));
+
+ // test backward compatibility with brackets ()
+ assertEquals(
+ Types.ROW_NAMED(
+ new String[] {"he \nllo", "world"},
+ Types.BIG_DEC, Types.BYTE),
+ TypeStringUtils.readTypeInfo("ROW(`he \nllo` DECIMAL, world TINYINT)"));
+
+ // test nesting
+ testReadAndWrite(
+ "ROW<singleton ROW<f0 INT>, twoField ROW<`Field 1` ROW<f0 DECIMAL>, `Field``s 2` VARCHAR>>",
+ Types.ROW_NAMED(
+ new String[] {"singleton", "twoField"},
+ Types.ROW(Types.INT),
+ Types.ROW_NAMED(
+ new String[] {"Field 1", "Field`s 2"},
+ Types.ROW(Types.BIG_DEC), Types.STRING)));
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testSyntaxError1() {
+ TypeStringUtils.readTypeInfo("ROW<<f0 DECIMAL, f1 TINYINT>"); // additional <
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testSyntaxError2() {
+ TypeStringUtils.readTypeInfo("ROW<f0 DECIMAL DECIMAL, f1 TINYINT>"); // duplicate type
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testSyntaxError3() {
+ TypeStringUtils.readTypeInfo("ROW<f0 INVALID, f1 TINYINT>"); // invalid type
+ }
+
+ private void testReadAndWrite(String expected, TypeInformation<?> type) {
+ // test read from string
+ assertEquals(type, TypeStringUtils.readTypeInfo(expected));
+
+ // test write to string
+ assertEquals(expected, TypeStringUtils.writeTypeInfo(type));
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Test POJO.
+ */
+ public static class TestPojo {
+ public int field1;
+ public String field2;
+ }
+
+ /**
+ * Test invalid POJO.
+ */
+ public static class TestNoPojo {
+ public int field1;
+ public String field2;
+
+ public TestNoPojo(int field1, String field2) { // no default constructor
+ this.field1 = field1;
+ this.field2 = field2;
+ }
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
index e64410e..6288ca5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
@@ -22,7 +22,7 @@ import java.io.Serializable
import java.lang.{Boolean => JBoolean, Byte => JByte, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong, Short => JShort}
import java.math.{BigDecimal => JBigDecimal}
import java.util
-import java.util.function.{BiConsumer, Consumer, Supplier, Function}
+import java.util.function.{BiConsumer, Consumer, Function, Supplier}
import java.util.regex.Pattern
import java.util.{Optional, List => JList, Map => JMap}
@@ -34,9 +34,9 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.configuration.MemorySize
import org.apache.flink.table.api.{TableException, TableSchema, ValidationException}
import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema, toJava}
-import org.apache.flink.table.typeutils.TypeStringUtils
-import org.apache.flink.util.{InstantiationUtil, Preconditions}
+import org.apache.flink.table.utils.TypeStringUtils
import org.apache.flink.util.Preconditions.checkNotNull
+import org.apache.flink.util.{InstantiationUtil, Preconditions}
import scala.collection.JavaConverters._
import scala.collection.mutable
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
index ac86ba7..8f3ff77 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValue.scala
@@ -20,7 +20,7 @@ package org.apache.flink.table.descriptors
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.typeutils.TypeStringUtils
+import org.apache.flink.table.utils.TypeStringUtils
import org.apache.flink.util.Preconditions
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeStringUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeStringUtils.scala
deleted file mode 100644
index 9e5f075..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/typeutils/TypeStringUtils.scala
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.typeutils
-
-import java.io.Serializable
-
-import org.apache.commons.codec.binary.Base64
-import org.apache.commons.lang3.StringEscapeUtils
-import org.apache.flink.api.common.functions.InvalidTypesException
-import org.apache.flink.api.common.typeinfo.{PrimitiveArrayTypeInfo, TypeInformation}
-import org.apache.flink.api.common.typeutils.CompositeType
-import org.apache.flink.api.java.typeutils._
-import org.apache.flink.table.api.{TableException, Types, ValidationException}
-import org.apache.flink.table.descriptors.DescriptorProperties.normalizeTypeInfo
-import org.apache.flink.util.InstantiationUtil
-
-import _root_.scala.language.implicitConversions
-import _root_.scala.util.parsing.combinator.{JavaTokenParsers, PackratParsers}
-
-/**
- * Utilities to convert [[org.apache.flink.api.common.typeinfo.TypeInformation]] into a
- * string representation and back.
- */
-object TypeStringUtils extends JavaTokenParsers with PackratParsers {
- case class Keyword(key: String)
-
- // convert the keyword into an case insensitive Parser
- implicit def keyword2Parser(kw: Keyword): Parser[String] = {
- ("""(?i)\Q""" + kw.key + """\E""").r
- }
-
- lazy val VARCHAR: Keyword = Keyword("VARCHAR")
- lazy val STRING: Keyword = Keyword("STRING")
- lazy val BOOLEAN: Keyword = Keyword("BOOLEAN")
- lazy val BYTE: Keyword = Keyword("BYTE")
- lazy val TINYINT: Keyword = Keyword("TINYINT")
- lazy val SHORT: Keyword = Keyword("SHORT")
- lazy val SMALLINT: Keyword = Keyword("SMALLINT")
- lazy val INT: Keyword = Keyword("INT")
- lazy val LONG: Keyword = Keyword("LONG")
- lazy val BIGINT: Keyword = Keyword("BIGINT")
- lazy val FLOAT: Keyword = Keyword("FLOAT")
- lazy val DOUBLE: Keyword = Keyword("DOUBLE")
- lazy val DECIMAL: Keyword = Keyword("DECIMAL")
- lazy val SQL_DATE: Keyword = Keyword("SQL_DATE")
- lazy val DATE: Keyword = Keyword("DATE")
- lazy val SQL_TIME: Keyword = Keyword("SQL_TIME")
- lazy val TIME: Keyword = Keyword("TIME")
- lazy val SQL_TIMESTAMP: Keyword = Keyword("SQL_TIMESTAMP")
- lazy val TIMESTAMP: Keyword = Keyword("TIMESTAMP")
- lazy val ROW: Keyword = Keyword("ROW")
- lazy val ANY: Keyword = Keyword("ANY")
- lazy val POJO: Keyword = Keyword("POJO")
- lazy val MAP: Keyword = Keyword("MAP")
- lazy val MULTISET: Keyword = Keyword("MULTISET")
- lazy val PRIMITIVE_ARRAY: Keyword = Keyword("PRIMITIVE_ARRAY")
- lazy val OBJECT_ARRAY: Keyword = Keyword("OBJECT_ARRAY")
-
- lazy val qualifiedName: Parser[String] =
- """\p{javaJavaIdentifierStart}[\p{javaJavaIdentifierPart}.]*""".r
-
- lazy val base64Url: Parser[String] =
- """[A-Za-z0-9_-]*""".r
-
- // keep parenthesis to ensure backward compatibility (this can be dropped after Flink 1.7)
- lazy val leftBracket: PackratParser[(String)] =
- "(" | "<"
-
- lazy val rightBracket: PackratParser[(String)] =
- ")" | ">"
-
- lazy val atomic: PackratParser[TypeInformation[_]] =
- (VARCHAR | STRING) ^^ { e => Types.STRING } |
- BOOLEAN ^^ { e => Types.BOOLEAN } |
- (TINYINT | BYTE) ^^ { e => Types.BYTE } |
- (SMALLINT | SHORT) ^^ { e => Types.SHORT } |
- INT ^^ { e => Types.INT } |
- (BIGINT | LONG) ^^ { e => Types.LONG } |
- FLOAT ^^ { e => Types.FLOAT } |
- DOUBLE ^^ { e => Types.DOUBLE } |
- DECIMAL ^^ { e => Types.DECIMAL } |
- (DATE | SQL_DATE) ^^ { e => Types.SQL_DATE.asInstanceOf[TypeInformation[_]] } |
- (TIMESTAMP | SQL_TIMESTAMP) ^^ { e => Types.SQL_TIMESTAMP } |
- (TIME | SQL_TIME) ^^ { e => Types.SQL_TIME }
-
- lazy val escapedFieldName: PackratParser[String] = stringLiteral ^^ { s =>
- val unquoted = s.substring(1, s.length - 1)
- StringEscapeUtils.unescapeJava(unquoted)
- }
-
- lazy val fieldName: PackratParser[String] = escapedFieldName | ident
-
- lazy val field: PackratParser[(String, TypeInformation[_])] =
- fieldName ~ typeInfo ^^ {
- case name ~ info => (name, info)
- }
-
- lazy val namedRow: PackratParser[TypeInformation[_]] =
- ROW ~ leftBracket ~> rep1sep(field, ", ") <~ rightBracket ^^ {
- fields => Types.ROW(fields.map(_._1).toArray, fields.map(_._2).toArray)
- } | failure("Named row type expected.")
-
- lazy val unnamedRow: PackratParser[TypeInformation[_]] =
- ROW ~ leftBracket ~> rep1sep(typeInfo, ", ") <~ rightBracket ^^ {
- types => Types.ROW(types: _*)
- } | failure("Unnamed row type expected.")
-
- lazy val generic: PackratParser[TypeInformation[_]] =
- ANY ~ leftBracket ~> qualifiedName <~ rightBracket ^^ {
- typeClass =>
- val clazz = loadClass(typeClass)
- new GenericTypeInfo[AnyRef](clazz.asInstanceOf[Class[AnyRef]])
- }
-
- lazy val pojo: PackratParser[TypeInformation[_]] =
- POJO ~ leftBracket ~> qualifiedName <~ rightBracket ^^ {
- typeClass =>
- val clazz = loadClass(typeClass)
- val info = TypeExtractor.createTypeInfo(clazz)
- if (!info.isInstanceOf[PojoTypeInfo[_]]) {
- throw new ValidationException(s"Class '$typeClass'is not a POJO type.")
- }
- info
- }
-
- lazy val any: PackratParser[TypeInformation[_]] =
- ANY ~ leftBracket ~ qualifiedName ~ "," ~ base64Url ~ rightBracket ^^ {
- case _ ~ _ ~ typeClass ~ _ ~ serialized ~ _=>
- val clazz = loadClass(typeClass)
- val typeInfo = deserialize(serialized)
-
- if (clazz != typeInfo.getTypeClass) {
- throw new ValidationException(
- s"Class '$typeClass' does no correspond to serialized data.")
- }
- typeInfo
- }
-
- lazy val genericMap: PackratParser[TypeInformation[_]] =
- MAP ~ leftBracket ~ typeInfo ~ "," ~ typeInfo ~ rightBracket ^^ {
- case _ ~ _ ~ keyTypeInfo ~ _ ~ valueTypeInfo ~ _=>
- Types.MAP(keyTypeInfo, valueTypeInfo)
- }
-
- lazy val multiSet: PackratParser[TypeInformation[_]] =
- MULTISET ~ leftBracket ~ typeInfo ~ rightBracket ^^ {
- case _ ~ _ ~ elementTypeInfo ~ _ =>
- Types.MULTISET(elementTypeInfo)
- }
-
- lazy val primitiveArray: PackratParser[TypeInformation[_]] =
- PRIMITIVE_ARRAY ~ leftBracket ~ typeInfo ~ rightBracket ^^ {
- case _ ~ _ ~ componentTypeInfo ~ _ =>
- Types.PRIMITIVE_ARRAY(componentTypeInfo)
- }
-
- lazy val objectArray: PackratParser[TypeInformation[_]] =
- OBJECT_ARRAY ~ leftBracket ~ typeInfo ~ rightBracket ^^ {
- case _ ~ _ ~ componentTypeInfo ~ _ =>
- Types.OBJECT_ARRAY(componentTypeInfo)
- }
-
- lazy val map: PackratParser[TypeInformation[_]] =
- genericMap | multiSet
-
- lazy val array: PackratParser[TypeInformation[_]] =
- primitiveArray | objectArray
-
- lazy val typeInfo: PackratParser[TypeInformation[_]] =
- namedRow | unnamedRow | any | generic | pojo | atomic | map | array | failure("Invalid type.")
-
- def readTypeInfo(typeString: String): TypeInformation[_] = {
- parseAll(typeInfo, typeString) match {
- case Success(lst, _) => lst
-
- case NoSuccess(msg, next) =>
- throwError(msg, next)
- }
- }
-
- // ----------------------------------------------------------------------------------------------
-
- def writeTypeInfo(typeInfo: TypeInformation[_]): String = typeInfo match {
-
- case Types.STRING => VARCHAR.key
- case Types.BOOLEAN => BOOLEAN.key
- case Types.BYTE => TINYINT.key
- case Types.SHORT => SMALLINT.key
- case Types.INT => INT.key
- case Types.LONG => BIGINT.key
- case Types.FLOAT => FLOAT.key
- case Types.DOUBLE => DOUBLE.key
- case Types.DECIMAL => DECIMAL.key
- case Types.SQL_DATE => DATE.key
- case Types.SQL_TIME => TIME.key
- case Types.SQL_TIMESTAMP => TIMESTAMP.key
-
- case rt: RowTypeInfo =>
- val fields = rt.getFieldNames.zip(rt.getFieldTypes)
- val normalizedFields = fields.map { f =>
-
- // escape field name if it contains spaces
- val name = if (!f._1.matches("\\S+")) {
- "\"" + StringEscapeUtils.escapeJava(f._1) + "\""
- } else {
- f._1
- }
-
- s"$name ${normalizeTypeInfo(f._2)}"
- }
- s"${ROW.key}<${normalizedFields.mkString(", ")}>"
-
- case generic: GenericTypeInfo[_] =>
- s"${ANY.key}<${generic.getTypeClass.getName}>"
-
- case pojo: PojoTypeInfo[_] =>
- // we only support very simple POJOs that only contain extracted fields
- // (not manually specified)
- val extractedInfo = try {
- Some(TypeExtractor.createTypeInfo(pojo.getTypeClass))
- } catch {
- case _: InvalidTypesException => None
- }
- extractedInfo match {
- case Some(ei) if ei == pojo => s"${POJO.key}<${pojo.getTypeClass.getName}>"
- case _ =>
- throw new TableException(
- "A string representation for custom POJO types is not supported yet.")
- }
-
- case array: PrimitiveArrayTypeInfo[_] =>
- s"${PRIMITIVE_ARRAY.key}<${normalizeTypeInfo(array.getComponentType)}>"
-
- case array: ObjectArrayTypeInfo[_, _] =>
- s"${OBJECT_ARRAY.key}<${normalizeTypeInfo(array.getComponentInfo)}>"
-
- case set: MultisetTypeInfo[_] =>
- s"${MULTISET.key}<${normalizeTypeInfo(set.getElementTypeInfo)}>"
-
- case map: MapTypeInfo[_, _] =>
- val normalizedKey = normalizeTypeInfo(map.getKeyTypeInfo)
- val normalizedValue = normalizeTypeInfo(map.getValueTypeInfo)
- s"${MAP.key}<$normalizedKey, $normalizedValue>"
-
- case any: TypeInformation[_] =>
- s"${ANY.key}<${any.getTypeClass.getName}, ${serialize(any)}>"
- }
-
- // ----------------------------------------------------------------------------------------------
-
- private def serialize(obj: Serializable): String = {
- try {
- val byteArray = InstantiationUtil.serializeObject(obj)
- Base64.encodeBase64URLSafeString(byteArray)
- } catch {
- case e: Exception =>
- throw new ValidationException(s"Unable to serialize type information '$obj' with " +
- s"class '${obj.getClass.getName}'.", e)
- }
- }
-
- private def deserialize(data: String): TypeInformation[_] = {
- val byteData = Base64.decodeBase64(data)
- InstantiationUtil
- .deserializeObject[TypeInformation[_]](byteData, Thread.currentThread.getContextClassLoader)
- }
-
- private def throwError(msg: String, next: Input): Nothing = {
- val improvedMsg = msg.replace("string matching regex `\\z'", "End of type information")
-
- throw new ValidationException(
- s"""Could not parse type information at column ${next.pos.column}: $improvedMsg
- |${next.pos.longString}""".stripMargin)
- }
-
- private def loadClass(qualifiedName: String): Class[_] = try {
- Class.forName(qualifiedName, true, Thread.currentThread.getContextClassLoader)
- } catch {
- case e: ClassNotFoundException =>
- throw new ValidationException("Class '" + qualifiedName + "' could not be found. " +
- "Please note that inner classes must be globally accessible and declared static.", e)
- }
-}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/typeutils/TypeStringUtilsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/typeutils/TypeStringUtilsTest.scala
deleted file mode 100644
index f35b0e0..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/typeutils/TypeStringUtilsTest.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.typeutils
-
-import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
-import org.apache.flink.api.java.typeutils.{RowTypeInfo, TypeExtractor}
-import org.apache.flink.table.api.Types
-import org.apache.flink.table.runtime.utils.CommonTestData.{NonPojo, Person}
-import org.junit.Assert.{assertEquals, assertTrue}
-import org.junit.Test
-
-/**
- * Tests for string-based representation of [[TypeInformation]].
- */
-class TypeStringUtilsTest {
-
- @Test
- def testPrimitiveTypes(): Unit = {
- testReadAndWrite("VARCHAR", Types.STRING)
- testReadAndWrite("BOOLEAN", Types.BOOLEAN)
- testReadAndWrite("TINYINT", Types.BYTE)
- testReadAndWrite("SMALLINT", Types.SHORT)
- testReadAndWrite("INT", Types.INT)
- testReadAndWrite("BIGINT", Types.LONG)
- testReadAndWrite("FLOAT", Types.FLOAT)
- testReadAndWrite("DOUBLE", Types.DOUBLE)
- testReadAndWrite("DECIMAL", Types.DECIMAL)
- testReadAndWrite("DATE", Types.SQL_DATE)
- testReadAndWrite("TIME", Types.SQL_TIME)
- testReadAndWrite("TIMESTAMP", Types.SQL_TIMESTAMP)
-
- // unsupported type information
- testReadAndWrite(
- "ANY<java.lang.Void, " +
- "rO0ABXNyADJvcmcuYXBhY2hlLmZsaW5rLmFwaS5jb21tb24udHlwZWluZm8uQmFzaWNUeXBlSW5mb_oE8IKl" +
- "ad0GAgAETAAFY2xhenp0ABFMamF2YS9sYW5nL0NsYXNzO0wAD2NvbXBhcmF0b3JDbGFzc3EAfgABWwAXcG9z" +
- "c2libGVDYXN0VGFyZ2V0VHlwZXN0ABJbTGphdmEvbGFuZy9DbGFzcztMAApzZXJpYWxpemVydAA2TG9yZy9h" +
- "cGFjaGUvZmxpbmsvYXBpL2NvbW1vbi90eXBldXRpbHMvVHlwZVNlcmlhbGl6ZXI7eHIANG9yZy5hcGFjaGUu" +
- "ZmxpbmsuYXBpLmNvbW1vbi50eXBlaW5mby5UeXBlSW5mb3JtYXRpb26UjchIurN66wIAAHhwdnIADmphdmEu" +
- "bGFuZy5Wb2lkAAAAAAAAAAAAAAB4cHB1cgASW0xqYXZhLmxhbmcuQ2xhc3M7qxbXrsvNWpkCAAB4cAAAAABz" +
- "cgA5b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1dGlscy5iYXNlLlZvaWRTZXJpYWxpemVyAAAA" +
- "AAAAAAECAAB4cgBCb3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1dGlscy5iYXNlLlR5cGVTZXJp" +
- "YWxpemVyU2luZ2xldG9ueamHqscud0UCAAB4cgA0b3JnLmFwYWNoZS5mbGluay5hcGkuY29tbW9uLnR5cGV1" +
- "dGlscy5UeXBlU2VyaWFsaXplcgAAAAAAAAABAgAAeHA>",
- BasicTypeInfo.VOID_TYPE_INFO)
- }
-
- @Test
- def testWriteComplexTypes(): Unit = {
- testReadAndWrite(
- "ROW<f0 DECIMAL, f1 TINYINT>",
- Types.ROW(Types.DECIMAL, Types.BYTE))
-
- testReadAndWrite(
- "ROW<hello DECIMAL, world TINYINT>",
- Types.ROW(
- Array[String]("hello", "world"),
- Array[TypeInformation[_]](Types.DECIMAL, Types.BYTE)))
-
- testReadAndWrite(
- "POJO<org.apache.flink.table.runtime.utils.CommonTestData$Person>",
- TypeExtractor.createTypeInfo(classOf[Person]))
-
- testReadAndWrite(
- "ANY<org.apache.flink.table.runtime.utils.CommonTestData$NonPojo>",
- TypeExtractor.createTypeInfo(classOf[NonPojo]))
-
- testReadAndWrite(
- "MAP<VARCHAR, ROW<f0 DECIMAL, f1 TINYINT>>",
- Types.MAP(Types.STRING, Types.ROW(Types.DECIMAL, Types.BYTE))
- )
-
- testReadAndWrite(
- "MULTISET<ROW<f0 DECIMAL, f1 TINYINT>>",
- Types.MULTISET(Types.ROW(Types.DECIMAL, Types.BYTE))
- )
-
- testReadAndWrite(
- "PRIMITIVE_ARRAY<TINYINT>",
- Types.PRIMITIVE_ARRAY(Types.BYTE)
- )
-
- testReadAndWrite(
- "OBJECT_ARRAY<POJO<org.apache.flink.table.runtime.utils.CommonTestData$Person>>",
- Types.OBJECT_ARRAY(TypeExtractor.createTypeInfo(classOf[Person]))
- )
-
- // test escaping
- assertTrue(
- TypeStringUtils.readTypeInfo("ROW<\"he \\nllo\" DECIMAL, world TINYINT>")
- .asInstanceOf[RowTypeInfo].getFieldNames
- .sameElements(Array[String]("he \nllo", "world")))
-
- // test backward compatibility with brackets ()
- assertTrue(
- TypeStringUtils.readTypeInfo("ROW(\"he \\nllo\" DECIMAL, world TINYINT)")
- .asInstanceOf[RowTypeInfo].getFieldNames
- .sameElements(Array[String]("he \nllo", "world")))
- }
-
- private def testReadAndWrite(expected: String, tpe: TypeInformation[_]): Unit = {
- // test write to string
- assertEquals(expected, TypeStringUtils.writeTypeInfo(tpe))
-
- // test read from string
- assertEquals(tpe, TypeStringUtils.readTypeInfo(expected))
- }
-}
[flink] 01/07: [FLINK-10687] [table] Introduce flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f3bce3cb99bf6a754f472a1151ac642e4d51f7d8
Author: Timo Walther <tw...@apache.org>
AuthorDate: Fri Oct 26 11:12:11 2018 +0200
[FLINK-10687] [table] Introduce flink-table-common
Because more and more table factories for connectors and formats are
added and external catalog support is also on the horizon, flink-table
becomes a dependency for many Flink modules. Since flink-table is
implemented in Scala it requires other modules to be suffixes with
Scala prefixes. However, as we have learned in the past, Scala code
is hard to maintain which is why our long-term goal is to avoid
Scala/Scala dependencies.
This commits adds a new module flink-table-common that contains
interfaces between flink-table and other modules. This module is
implemented in Java and should contain minimal (or better no)
external dependencies.
---
flink-libraries/flink-table-common/pom.xml | 58 ++++++++++++++++++++++++++++++
flink-libraries/flink-table/pom.xml | 6 ++++
flink-libraries/pom.xml | 1 +
3 files changed, 65 insertions(+)
diff --git a/flink-libraries/flink-table-common/pom.xml b/flink-libraries/flink-table-common/pom.xml
new file mode 100644
index 0000000..2ebc7f9
--- /dev/null
+++ b/flink-libraries/flink-table-common/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-libraries</artifactId>
+ <version>1.7-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-table-common</artifactId>
+ <name>flink-table-common</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>1.7-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index c3dab6e..c7b3c55 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -61,6 +61,12 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
diff --git a/flink-libraries/pom.xml b/flink-libraries/pom.xml
index 9529be6..0e98fbf 100644
--- a/flink-libraries/pom.xml
+++ b/flink-libraries/pom.xml
@@ -39,6 +39,7 @@ under the License.
<module>flink-gelly-examples</module>
<module>flink-python</module>
<module>flink-table</module>
+ <module>flink-table-common</module>
<module>flink-sql-client</module>
<module>flink-ml</module>
<module>flink-cep</module>
[flink] 02/07: [FLINK-10687] [table] Move TableException and
ValidationException to flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f0b244e15720d698a486d778085cc5ab6ceb5f73
Author: Timo Walther <tw...@apache.org>
AuthorDate: Fri Oct 26 13:35:17 2018 +0200
[FLINK-10687] [table] Move TableException and ValidationException to flink-table-common
---
.../org/apache/flink/table/api/TableException.java | 39 ++++++++++++++++++++++
.../flink/table/api/ValidationException.java | 38 +++++++++++++++++++++
.../flink/table/api/BatchTableEnvironment.scala | 6 ++--
.../flink/table/api/StreamTableEnvironment.scala | 10 +++---
.../apache/flink/table/api/TableEnvironment.scala | 17 +++++-----
.../org/apache/flink/table/api/exceptions.scala | 37 --------------------
.../scala/org/apache/flink/table/api/table.scala | 16 ++++-----
.../flink/table/calcite/FlinkPlannerImpl.scala | 2 +-
.../flink/table/calcite/FlinkTypeFactory.scala | 13 ++++----
.../apache/flink/table/codegen/CodeGenerator.scala | 2 +-
.../table/descriptors/LiteralValueValidator.scala | 4 +--
.../flink/table/expressions/ExpressionUtils.scala | 2 +-
.../org/apache/flink/table/expressions/call.scala | 2 +-
.../flink/table/expressions/fieldExpression.scala | 2 +-
.../functions/utils/UserDefinedFunctionUtils.scala | 9 ++---
.../flink/table/plan/nodes/CommonCorrelate.scala | 2 +-
.../flink/table/plan/nodes/FlinkRelNode.scala | 2 +-
.../table/plan/nodes/dataset/DataSetJoin.scala | 10 +++---
.../table/plan/nodes/dataset/DataSetSort.scala | 2 +-
.../DataStreamGroupWindowAggregate.scala | 2 +-
.../plan/nodes/datastream/DataStreamJoin.scala | 2 +-
.../plan/nodes/datastream/DataStreamMatch.scala | 12 +++----
.../rules/common/LogicalWindowAggregateRule.scala | 6 ++--
.../plan/rules/common/WindowPropertiesRule.scala | 5 +--
.../rules/dataSet/DataSetWindowAggregateRule.scala | 4 +--
.../datastream/DataStreamGroupAggregateRule.scala | 2 +-
.../DataStreamGroupWindowAggregateRule.scala | 2 +-
.../DataStreamLogicalWindowAggregateRule.scala | 6 ++--
.../plan/rules/logical/LogicalUnnestRule.scala | 4 +--
.../table/runtime/aggregate/AggregateUtil.scala | 13 +++++---
.../flink/table/sources/CsvTableSource.scala | 2 +-
.../flink/table/sources/TableSourceUtil.scala | 27 +++++++--------
.../table/sources/tsextractors/ExistingField.scala | 2 +-
.../flink/table/validate/FunctionCatalog.scala | 14 ++++----
.../flink/table/descriptors/RowtimeTest.scala | 2 +-
35 files changed, 184 insertions(+), 136 deletions(-)
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableException.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableException.java
new file mode 100644
index 0000000..caef3a8
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/TableException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * General Exception for all errors during table handling.
+ *
+ * <p>This exception indicates that an internal error occurred or that a feature is not supported
+ * yet. Usually, this exception does not indicate a fault of the user.
+ */
+@PublicEvolving
+public class TableException extends RuntimeException {
+
+ public TableException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TableException(String message) {
+ super(message);
+ }
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/ValidationException.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/ValidationException.java
new file mode 100644
index 0000000..c6ac1dc
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/api/ValidationException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * Exception for all errors occurring during validation phase.
+ *
+ * <p>This exception indicates that the user did something wrong.
+ */
+@PublicEvolving
+public class ValidationException extends RuntimeException {
+
+ public ValidationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public ValidationException(String message) {
+ super(message);
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
index 5a34ee1..6a7a921 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/BatchTableEnvironment.scala
@@ -205,8 +205,8 @@ abstract class BatchTableEnvironment(
tableSink: TableSink[_]): Unit = {
// validate
checkValidTableName(name)
- if (fieldNames == null) throw TableException("fieldNames must not be null.")
- if (fieldTypes == null) throw TableException("fieldTypes must not be null.")
+ if (fieldNames == null) throw new TableException("fieldNames must not be null.")
+ if (fieldTypes == null) throw new TableException("fieldTypes must not be null.")
if (fieldNames.length == 0) throw new TableException("fieldNames must not be empty.")
if (fieldNames.length != fieldTypes.length) {
throw new TableException("Same number of field names and types required.")
@@ -513,7 +513,7 @@ abstract class BatchTableEnvironment(
}
case _ =>
- throw TableException("Cannot generate DataSet due to an invalid logical plan. " +
+ throw new TableException("Cannot generate DataSet due to an invalid logical plan. " +
"This is a bug and should not happen. Please file an issue.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
index 6519120..4973f34 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/StreamTableEnvironment.scala
@@ -120,7 +120,7 @@ abstract class StreamTableEnvironment(
// check that event-time is enabled if table source includes rowtime attributes
if (TableSourceUtil.hasRowtimeAttribute(streamTableSource) &&
execEnv.getStreamTimeCharacteristic != TimeCharacteristic.EventTime) {
- throw TableException(
+ throw new TableException(
s"A rowtime attribute requires an EventTime time characteristic in stream " +
s"environment. But is: ${execEnv.getStreamTimeCharacteristic}")
}
@@ -230,8 +230,8 @@ abstract class StreamTableEnvironment(
tableSink: TableSink[_]): Unit = {
checkValidTableName(name)
- if (fieldNames == null) throw TableException("fieldNames must not be null.")
- if (fieldTypes == null) throw TableException("fieldTypes must not be null.")
+ if (fieldNames == null) throw new TableException("fieldNames must not be null.")
+ if (fieldTypes == null) throw new TableException("fieldTypes must not be null.")
if (fieldNames.length == 0) throw new TableException("fieldNames must not be empty.")
if (fieldNames.length != fieldTypes.length) {
throw new TableException("Same number of field names and types required.")
@@ -550,7 +550,7 @@ abstract class StreamTableEnvironment(
// check if event-time is enabled
if (rowtime.isDefined && execEnv.getStreamTimeCharacteristic != TimeCharacteristic.EventTime) {
- throw TableException(
+ throw new TableException(
s"A rowtime attribute requires an EventTime time characteristic in stream environment. " +
s"But is: ${execEnv.getStreamTimeCharacteristic}")
}
@@ -966,7 +966,7 @@ abstract class StreamTableEnvironment(
case node: DataStreamRel =>
node.translateToPlan(this, queryConfig)
case _ =>
- throw TableException("Cannot generate DataStream due to an invalid logical plan. " +
+ throw new TableException("Cannot generate DataStream due to an invalid logical plan. " +
"This is a bug and should not happen. Please file an issue.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
index 7cfe249..3cc97d3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableEnvironment.scala
@@ -356,7 +356,7 @@ abstract class TableEnvironment(val config: TableConfig) {
throw new TableException(
s"Cannot generate a valid execution plan for the given query: \n\n" +
s"${RelOptUtil.toString(input)}\n" +
- s"${t.msg}\n" +
+ s"${t.getMessage}\n" +
s"Please check the documentation for the set of currently supported SQL features.")
case a: AssertionError =>
// keep original exception stack for caller
@@ -817,13 +817,13 @@ abstract class TableEnvironment(val config: TableConfig) {
private[flink] def insertInto(table: Table, sinkTableName: String, conf: QueryConfig): Unit = {
// check that sink table exists
- if (null == sinkTableName) throw TableException("Name of TableSink must not be null.")
- if (sinkTableName.isEmpty) throw TableException("Name of TableSink must not be empty.")
+ if (null == sinkTableName) throw new TableException("Name of TableSink must not be null.")
+ if (sinkTableName.isEmpty) throw new TableException("Name of TableSink must not be empty.")
getTable(sinkTableName) match {
case None =>
- throw TableException(s"No table was registered under the name $sinkTableName.")
+ throw new TableException(s"No table was registered under the name $sinkTableName.")
case Some(s: TableSourceSinkTable[_, _]) if s.tableSinkTable.isDefined =>
val tableSink = s.tableSinkTable.get.tableSink
@@ -845,7 +845,7 @@ abstract class TableEnvironment(val config: TableConfig) {
.map { case (n, t) => s"$n: ${t.getTypeClass.getSimpleName}" }
.mkString("[", ", ", "]")
- throw ValidationException(
+ throw new ValidationException(
s"Field types of query result and registered TableSink " +
s"$sinkTableName do not match.\n" +
s"Query result schema: $srcSchema\n" +
@@ -855,7 +855,7 @@ abstract class TableEnvironment(val config: TableConfig) {
writeToSink(table, tableSink, conf)
case Some(_) =>
- throw TableException(s"The table registered as $sinkTableName is not a TableSink. " +
+ throw new TableException(s"The table registered as $sinkTableName is not a TableSink. " +
s"You can only emit query results to a registered TableSink.")
}
}
@@ -1119,7 +1119,7 @@ abstract class TableEnvironment(val config: TableConfig) {
// validate that at least the field types of physical and logical type match
// we do that here to make sure that plan translation was correct
if (schema.typeInfo != inputTypeInfo) {
- throw TableException(
+ throw new TableException(
s"The field types of physical and logical row types do not match. " +
s"Physical type is [${schema.typeInfo}], Logical type is [$inputTypeInfo]. " +
s"This is a bug and should not happen. Please file an issue.")
@@ -1350,7 +1350,8 @@ object TableEnvironment {
if ((clazz.isMemberClass && !Modifier.isStatic(clazz.getModifiers)) ||
!Modifier.isPublic(clazz.getModifiers) ||
clazz.getCanonicalName == null) {
- throw TableException(s"Class '$clazz' described in type information '$typeInfo' must be " +
+ throw new TableException(
+ s"Class '$clazz' described in type information '$typeInfo' must be " +
s"static and globally accessible.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
index 81dad3e..0ca17a5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
@@ -39,43 +39,6 @@ case class SqlParserException(
}
/**
- * General Exception for all errors during table handling.
- *
- * This exception indicates that an internal error occurred or that a feature is not supported
- * yet. Usually, this exception does not indicate a fault of the user.
- */
-case class TableException(
- msg: String,
- cause: Throwable)
- extends RuntimeException(msg, cause) {
-
- def this(msg: String) = this(msg, null)
-
-}
-
-object TableException {
- def apply(msg: String): TableException = new TableException(msg)
-}
-
-/**
- * Exception for all errors occurring during validation phase.
- *
- * This exception indicates that the user did something wrong.
- */
-case class ValidationException(
- msg: String,
- cause: Throwable)
- extends RuntimeException(msg, cause) {
-
- def this(msg: String) = this(msg, null)
-
-}
-
-object ValidationException {
- def apply(msg: String): ValidationException = new ValidationException(msg)
-}
-
-/**
* Exception for unwanted method calling on unresolved expression.
*/
case class UnresolvedException(msg: String) extends RuntimeException(msg)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
index 2b9570c..dfb5c0b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/table.scala
@@ -118,7 +118,7 @@ class Table(
val expandedFields = expandProjectList(fields, logicalPlan, tableEnv)
val (aggNames, propNames) = extractAggregationsAndProperties(expandedFields, tableEnv)
if (propNames.nonEmpty) {
- throw ValidationException("Window properties can only be used on windowed tables.")
+ throw new ValidationException("Window properties can only be used on windowed tables.")
}
if (aggNames.nonEmpty) {
@@ -822,14 +822,14 @@ class Table(
*/
def fetch(fetch: Int): Table = {
if (fetch < 0) {
- throw ValidationException("FETCH count must be equal or larger than 0.")
+ throw new ValidationException("FETCH count must be equal or larger than 0.")
}
this.logicalPlan match {
case Limit(o, -1, c) =>
// replace LIMIT without FETCH by LIMIT with FETCH
new Table(tableEnv, Limit(o, fetch, c).validate(tableEnv))
case Limit(_, _, _) =>
- throw ValidationException("FETCH is already defined.")
+ throw new ValidationException("FETCH is already defined.")
case _ =>
new Table(tableEnv, Limit(0, fetch, logicalPlan).validate(tableEnv))
}
@@ -934,7 +934,7 @@ class Table(
def insertInto(tableName: String, conf: QueryConfig): Unit = {
this.logicalPlan match {
case _: LogicalTableFunctionCall =>
- throw ValidationException("TableFunction can only be used in join and leftOuterJoin.")
+ throw new ValidationException("TableFunction can only be used in join and leftOuterJoin.")
case _ =>
tableEnv.insertInto(this, tableName, conf)
}
@@ -989,11 +989,11 @@ class Table(
def window(overWindows: OverWindow*): OverWindowedTable = {
if (tableEnv.isInstanceOf[BatchTableEnvironment]) {
- throw TableException("Over-windows for batch tables are currently not supported.")
+ throw new TableException("Over-windows for batch tables are currently not supported.")
}
if (overWindows.size != 1) {
- throw TableException("Over-Windows are currently only supported single window.")
+ throw new TableException("Over-Windows are currently only supported single window.")
}
new OverWindowedTable(this, overWindows.toArray)
@@ -1049,7 +1049,7 @@ class GroupedTable(
val expandedFields = expandProjectList(fields, table.logicalPlan, table.tableEnv)
val (aggNames, propNames) = extractAggregationsAndProperties(expandedFields, table.tableEnv)
if (propNames.nonEmpty) {
- throw ValidationException("Window properties can only be used on windowed tables.")
+ throw new ValidationException("Window properties can only be used on windowed tables.")
}
val projectsOnAgg = replaceAggregationsAndProperties(
@@ -1145,7 +1145,7 @@ class OverWindowedTable(
table.tableEnv)
if(fields.exists(_.isInstanceOf[WindowProperty])){
- throw ValidationException(
+ throw new ValidationException(
"Window start and end properties are not available for Over windows.")
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
index 2cd0c61..ca30053 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala
@@ -133,7 +133,7 @@ class FlinkPlannerImpl(
// root = root.withRel(RelTimeIndicatorConverter.convert(root.rel, rexBuilder))
root
} catch {
- case e: RelConversionException => throw TableException(e.getMessage)
+ case e: RelConversionException => throw new TableException(e.getMessage)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
index db7ffdb..c48967b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/calcite/FlinkTypeFactory.scala
@@ -172,7 +172,7 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
getTypeSystem.asInstanceOf[FlinkTypeSystem])
case ti@_ =>
- throw TableException(s"Unsupported type information: $ti")
+ throw new TableException(s"Unsupported type information: $ti")
}
canonize(relType)
@@ -307,7 +307,7 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp
if (allTypes.exists(_.getSqlTypeName == SqlTypeName.ANY)) {
// one of the type was ANY.
// we cannot generate a common type if it differs from other types.
- throw TableException("Generic ANY types must have a common type information.")
+ throw new TableException("Generic ANY types must have a common type information.")
} else {
// cannot resolve a common type for different input types
None
@@ -341,10 +341,10 @@ object FlinkTypeFactory {
case TimeIntervalTypeInfo.INTERVAL_MILLIS => INTERVAL_DAY_SECOND
case CHAR_TYPE_INFO | CHAR_VALUE_TYPE_INFO =>
- throw TableException("Character type is not supported.")
+ throw new TableException("Character type is not supported.")
case _@t =>
- throw TableException(s"Type is not supported: $t")
+ throw new TableException(s"Type is not supported: $t")
}
/**
@@ -419,7 +419,8 @@ object FlinkTypeFactory {
case typeName if DAY_INTERVAL_TYPES.contains(typeName) => TimeIntervalTypeInfo.INTERVAL_MILLIS
case NULL =>
- throw TableException("Type NULL is not supported. Null values must have a supported type.")
+ throw new TableException(
+ "Type NULL is not supported. Null values must have a supported type.")
// symbol for special flags e.g. TRIM's BOTH, LEADING, TRAILING
// are represented as integer
@@ -454,6 +455,6 @@ object FlinkTypeFactory {
multisetRelDataType.typeInfo
case _@t =>
- throw TableException(s"Type is not supported: $t")
+ throw new TableException(s"Type is not supported: $t")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
index 2816f11..5d047ad 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/codegen/CodeGenerator.scala
@@ -274,7 +274,7 @@ abstract class CodeGenerator(
generateExpression(rowtimeExpression.get)
case TimeIndicatorTypeInfo.ROWTIME_STREAM_MARKER |
TimeIndicatorTypeInfo.ROWTIME_BATCH_MARKER =>
- throw TableException("Rowtime extraction expression missing. Please report a bug.")
+ throw new TableException("Rowtime extraction expression missing. Please report a bug.")
case TimeIndicatorTypeInfo.PROCTIME_STREAM_MARKER =>
// attribute is proctime indicator.
// we use a null literal and generate a timestamp when we need it.
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
index 943b01a..f567131 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
@@ -62,7 +62,7 @@ class LiteralValueValidator(keyPrefix: String) extends HierarchyDescriptorValida
case Types.LONG => properties.validateLong(valueKey, isOptional = false)
case Types.SHORT => properties.validateShort(valueKey, isOptional = false)
case Types.STRING => properties.validateString(valueKey, isOptional = false)
- case _ => throw TableException(s"Unsupported type '$typeInfo'.")
+ case _ => throw new TableException(s"Unsupported type '$typeInfo'.")
}
}
// implicit type
@@ -107,7 +107,7 @@ object LiteralValueValidator {
case Types.LONG => properties.getLong(valueKey)
case Types.SHORT => properties.getShort(valueKey)
case Types.STRING => properties.getString(valueKey)
- case _ => throw TableException(s"Unsupported type '${typeInfo.getTypeClass}'.")
+ case _ => throw new TableException(s"Unsupported type '${typeInfo.getTypeClass}'.")
}
}
// implicit type
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
index e794d41..6080ba4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/ExpressionUtils.scala
@@ -138,7 +138,7 @@ object ExpressionUtils {
if (array.length > 0 && array.head.isInstanceOf[Array[_]]) {
ArrayConstructor(array.map { na => convertArray(na.asInstanceOf[Array[_]]) })
} else {
- throw ValidationException("Unsupported array type.")
+ throw new ValidationException("Unsupported array type.")
}
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
index 8454555..574cbb3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/call.scala
@@ -342,7 +342,7 @@ case class TableFunctionCall(
val fieldNames = if (aliases.isDefined) {
val aliasList = aliases.get
if (aliasList.length != originNames.length) {
- throw ValidationException(
+ throw new ValidationException(
s"List of column aliases must have same degree as table; " +
s"the returned table of function '$functionName' has ${originNames.length} " +
s"columns (${originNames.mkString(",")}), " +
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
index 3de0175..9721e62 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/expressions/fieldExpression.scala
@@ -190,7 +190,7 @@ case class RowtimeAttribute(expr: Expression) extends TimeAttribute(expr) {
// batch time window
Types.SQL_TIMESTAMP
case _ =>
- throw TableException("RowtimeAttribute has invalid type. Please report this bug.")
+ throw new TableException("RowtimeAttribute has invalid type. Please report this bug.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
index 2c08001..9799e4d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/utils/UserDefinedFunctionUtils.scala
@@ -53,15 +53,16 @@ object UserDefinedFunctionUtils {
*/
def checkForInstantiation(clazz: Class[_]): Unit = {
if (!InstantiationUtil.isPublic(clazz)) {
- throw ValidationException(s"Function class ${clazz.getCanonicalName} is not public.")
+ throw new ValidationException(s"Function class ${clazz.getCanonicalName} is not public.")
}
else if (!InstantiationUtil.isProperClass(clazz)) {
- throw ValidationException(s"Function class ${clazz.getCanonicalName} is no proper class," +
+ throw new ValidationException(
+ s"Function class ${clazz.getCanonicalName} is no proper class," +
" it is either abstract, an interface, or a primitive type.")
}
else if (InstantiationUtil.isNonStaticInnerClass(clazz)) {
- throw ValidationException(s"The class ${clazz.getCanonicalName} is an inner class, but" +
- " not statically accessible.")
+ throw new ValidationException(
+ s"The class ${clazz.getCanonicalName} is an inner class, but not statically accessible.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
index 3475e19..53adf6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/CommonCorrelate.scala
@@ -99,7 +99,7 @@ trait CommonCorrelate {
|}
|""".stripMargin
} else if (joinType != SemiJoinType.INNER) {
- throw TableException(s"Unsupported SemiJoinType: $joinType for correlate join.")
+ throw new TableException(s"Unsupported SemiJoinType: $joinType for correlate join.")
}
functionGenerator.generateFunction(
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
index ccfd904..56701f7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/FlinkRelNode.scala
@@ -103,6 +103,6 @@ trait FlinkRelNode extends RelNode {
// 16 is an arbitrary estimate
(estimateDataTypeSize(t.getKeyType) + estimateDataTypeSize(t.getValueType)) * 16
case SqlTypeName.ANY => 128 // 128 is an arbitrary estimate
- case _ => throw TableException(s"Unsupported data type encountered: $t")
+ case _ => throw new TableException(s"Unsupported data type encountered: $t")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetJoin.scala
index 56bf8ea..7f32e6d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetJoin.scala
@@ -128,7 +128,7 @@ class DataSetJoin(
val rightKeys = ArrayBuffer.empty[Int]
if (keyPairs.isEmpty) {
// if no equality keys => not supported
- throw TableException(
+ throw new TableException(
"Joins should have at least one equality condition.\n" +
s"\tLeft: ${left.toString},\n" +
s"\tRight: ${right.toString},\n" +
@@ -151,7 +151,7 @@ class DataSetJoin(
leftKeys.add(pair.source)
rightKeys.add(pair.target)
} else {
- throw TableException(
+ throw new TableException(
"Equality join predicate on incompatible types.\n" +
s"\tLeft: ${left.toString},\n" +
s"\tRight: ${right.toString},\n" +
@@ -255,7 +255,7 @@ class DataSetJoin(
config: TableConfig): DataSet[Row] = {
if (!config.getNullCheck) {
- throw TableException("Null check in TableConfig must be enabled for outer joins.")
+ throw new TableException("Null check in TableConfig must be enabled for outer joins.")
}
val joinOpName = getJoinOpName
@@ -311,7 +311,7 @@ class DataSetJoin(
config: TableConfig): DataSet[Row] = {
if (!config.getNullCheck) {
- throw TableException("Null check in TableConfig must be enabled for outer joins.")
+ throw new TableException("Null check in TableConfig must be enabled for outer joins.")
}
val joinOpName = getJoinOpName
@@ -367,7 +367,7 @@ class DataSetJoin(
config: TableConfig): DataSet[Row] = {
if (!config.getNullCheck) {
- throw TableException("Null check in TableConfig must be enabled for outer joins.")
+ throw new TableException("Null check in TableConfig must be enabled for outer joins.")
}
val joinOpName = getJoinOpName
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSort.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSort.scala
index 1430204..fae52fa 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSort.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetSort.scala
@@ -84,7 +84,7 @@ class DataSetSort(
queryConfig: BatchQueryConfig): DataSet[Row] = {
if (fieldCollations.isEmpty) {
- throw TableException("Limiting the result without sorting is not allowed " +
+ throw new TableException("Limiting the result without sorting is not allowed " +
"as it could lead to arbitrary results.")
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
index 0a014b6..78a7273 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamGroupWindowAggregate.scala
@@ -147,7 +147,7 @@ class DataStreamGroupWindowAggregate(
val timeAttribute = window.timeAttribute.asInstanceOf[ResolvedFieldReference].name
val timeIdx = inputSchema.fieldNames.indexOf(timeAttribute)
if (timeIdx < 0) {
- throw TableException("Time attribute could not be found. This is a bug.")
+ throw new TableException("Time attribute could not be found. This is a bug.")
}
inputDS
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamJoin.scala
index 3a38bc5..80c3123 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamJoin.scala
@@ -134,7 +134,7 @@ class DataStreamJoin(
val rightKeyType = rightFields.get(pair.target).getType.getSqlTypeName
// check if keys are compatible
if (leftKeyType != rightKeyType) {
- throw TableException(
+ throw new TableException(
"Equality join predicate on incompatible types.\n" +
s"\tLeft: $left,\n" +
s"\tRight: $right,\n" +
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
index 2bf6e32..19b298c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala
@@ -292,16 +292,16 @@ private[flink] class PatternVisitor(
applyQuantifier(pattern, startNum, endNum, isGreedy)
case PATTERN_ALTER =>
- throw TableException(s"Expression not supported: $call. Currently, CEP doesn't support " +
- s"branching patterns.")
+ throw new TableException(
+ s"Expression not supported: $call. Currently, CEP doesn't support branching patterns.")
case PATTERN_PERMUTE =>
- throw TableException(s"Expression not supported: $call. Currently, CEP doesn't support " +
- s"PERMUTE patterns.")
+ throw new TableException(
+ s"Expression not supported: $call. Currently, CEP doesn't support PERMUTE patterns.")
case PATTERN_EXCLUDE =>
- throw TableException(s"Expression not supported: $call. Currently, CEP doesn't support " +
- s"'{-' '-}' patterns.")
+ throw new TableException(
+ s"Expression not supported: $call. Currently, CEP doesn't support '{-' '-}' patterns.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/LogicalWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/LogicalWindowAggregateRule.scala
index 11701fc..8789ba6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/LogicalWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/LogicalWindowAggregateRule.scala
@@ -114,19 +114,19 @@ abstract class LogicalWindowAggregateRule(ruleName: String)
if (call.getOperands.size() == 2) {
true
} else {
- throw TableException("TUMBLE window with alignment is not supported yet.")
+ throw new TableException("TUMBLE window with alignment is not supported yet.")
}
case BasicOperatorTable.HOP =>
if (call.getOperands.size() == 3) {
true
} else {
- throw TableException("HOP window with alignment is not supported yet.")
+ throw new TableException("HOP window with alignment is not supported yet.")
}
case BasicOperatorTable.SESSION =>
if (call.getOperands.size() == 2) {
true
} else {
- throw TableException("SESSION window with alignment is not supported yet.")
+ throw new TableException("SESSION window with alignment is not supported yet.")
}
case _ => false
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowPropertiesRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowPropertiesRule.scala
index 74f7b1b..b9e34ff 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowPropertiesRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/common/WindowPropertiesRule.scala
@@ -145,7 +145,7 @@ abstract class WindowPropertiesBaseRule(rulePredicate: RelOptRuleOperand, ruleNa
// replace expression by access to window rowtime
builder.field(propertyName(window, "rowtime"))
case 'streamProctime =>
- throw ValidationException("A proctime window cannot provide a rowtime attribute.")
+ throw new ValidationException("A proctime window cannot provide a rowtime attribute.")
case _ =>
throw new TableException("Unknown window type encountered. Please report this bug.")
}
@@ -156,7 +156,8 @@ abstract class WindowPropertiesBaseRule(rulePredicate: RelOptRuleOperand, ruleNa
// replace expression by access to window proctime
builder.field(propertyName(window, "proctime"))
case 'batchRowtime =>
- throw ValidationException("PROCTIME window property is not supported in batch queries.")
+ throw new ValidationException(
+ "PROCTIME window property is not supported in batch queries.")
case _ =>
throw new TableException("Unknown window type encountered. Please report this bug.")
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetWindowAggregateRule.scala
index f215329..72d71aa 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetWindowAggregateRule.scala
@@ -40,13 +40,13 @@ class DataSetWindowAggregateRule
// check if we have distinct aggregates
val distinctAggs = agg.getAggCallList.exists(_.isDistinct)
if (distinctAggs) {
- throw TableException("DISTINCT aggregates are currently not supported.")
+ throw new TableException("DISTINCT aggregates are currently not supported.")
}
// check if we have grouping sets
val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
if (groupSets || agg.indicator) {
- throw TableException("GROUPING SETS are currently not supported.")
+ throw new TableException("GROUPING SETS are currently not supported.")
}
!distinctAggs && !groupSets && !agg.indicator
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
index e99a86e..c80a145 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupAggregateRule.scala
@@ -46,7 +46,7 @@ class DataStreamGroupAggregateRule
// check if we have grouping sets
val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
if (groupSets || agg.indicator) {
- throw TableException("GROUPING SETS are currently not supported.")
+ throw new TableException("GROUPING SETS are currently not supported.")
}
!groupSets && !agg.indicator
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
index b77b7bf..d48beaf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamGroupWindowAggregateRule.scala
@@ -42,7 +42,7 @@ class DataStreamGroupWindowAggregateRule
// check if we have grouping sets
val groupSets = agg.getGroupSets.size() != 1 || agg.getGroupSets.get(0) != agg.getGroupSet
if (groupSets || agg.indicator) {
- throw TableException("GROUPING SETS are currently not supported.")
+ throw new TableException("GROUPING SETS are currently not supported.")
}
!groupSets && !agg.indicator
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
index 254f36b..62eecbb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamLogicalWindowAggregateRule.scala
@@ -46,7 +46,7 @@ class DataStreamLogicalWindowAggregateRule
timeAttribute
case _ =>
- throw TableException(
+ throw new TableException(
s"Time attribute expected but ${timeAttribute.getType} encountered.")
}
}
@@ -69,7 +69,7 @@ class DataStreamLogicalWindowAggregateRule
def getOperandAsLong(call: RexCall, idx: Int): Long =
call.getOperands.get(idx) match {
case v: RexLiteral => v.getValue.asInstanceOf[JBigDecimal].longValue()
- case _ => throw TableException("Only constant window descriptors are supported.")
+ case _ => throw new TableException("Only constant window descriptors are supported.")
}
def getOperandAsTimeIndicator(call: RexCall, idx: Int): ResolvedFieldReference =
@@ -79,7 +79,7 @@ class DataStreamLogicalWindowAggregateRule
rowType.getFieldList.get(v.getIndex).getName,
FlinkTypeFactory.toTypeInfo(v.getType))
case _ =>
- throw ValidationException("Window can only be defined over a time attribute column.")
+ throw new ValidationException("Window can only be defined over a time attribute column.")
}
windowExpr.getOperator match {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
index f985a80..09886e0 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/plan/rules/logical/LogicalUnnestRule.scala
@@ -96,7 +96,7 @@ class LogicalUnnestRule(
ExplodeFunctionUtil.explodeTableFuncFromType(arrayType.typeInfo))
case mt: MultisetRelDataType =>
(mt.getComponentType, ExplodeFunctionUtil.explodeTableFuncFromType(mt.typeInfo))
- case _ => throw TableException(s"Unsupported UNNEST on type: ${dataType.toString}")
+ case _ => throw new TableException(s"Unsupported UNNEST on type: ${dataType.toString}")
}
// create sql function
@@ -121,7 +121,7 @@ class LogicalUnnestRule(
StructKind.FULLY_QUALIFIED,
ImmutableList.of(new RelDataTypeFieldImpl("f0", 0, componentType)))
case _: RelRecordType => componentType
- case _ => throw TableException(
+ case _ => throw new TableException(
s"Unsupported component type in UNNEST: ${componentType.toString}")
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
index a3c9c1e..1e2df6e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/AggregateUtil.scala
@@ -1146,15 +1146,18 @@ object AggregateUtil {
case NamedWindowProperty(_, prop) =>
prop match {
case WindowStart(_) if s.isDefined =>
- throw TableException("Duplicate window start property encountered. This is a bug.")
+ throw new TableException(
+ "Duplicate window start property encountered. This is a bug.")
case WindowStart(_) =>
(Some(i), e, rt, i - 1)
case WindowEnd(_) if e.isDefined =>
- throw TableException("Duplicate window end property encountered. This is a bug.")
+ throw new TableException(
+ "Duplicate window end property encountered. This is a bug.")
case WindowEnd(_) =>
(s, Some(i), rt, i - 1)
case RowtimeAttribute(_) if rt.isDefined =>
- throw TableException("Duplicate window rowtime property encountered. This is a bug.")
+ throw new TableException(
+ "Duplicate window rowtime property encountered. This is a bug.")
case RowtimeAttribute(_) =>
(s, e, Some(i), i - 1)
case ProctimeAttribute(_) =>
@@ -1550,10 +1553,10 @@ object AggregateUtil {
if (relDataType.length == 1) {
relDataType.head.getIndex
} else {
- throw TableException(
+ throw new TableException(
s"Encountered more than one time attribute with the same name: '$relDataType'")
}
- case e => throw TableException(
+ case e => throw new TableException(
"The time attribute of window in batch environment should be " +
s"ResolvedFieldReference, but is $e")
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSource.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSource.scala
index e0022c5..f7215ee 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSource.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSource.scala
@@ -115,7 +115,7 @@ class CsvTableSource private (
}
if (fieldNames.length != fieldTypes.length) {
- throw TableException("Number of field names and field types must be equal.")
+ throw new TableException("Number of field names and field types must be equal.")
}
private val selectedFieldTypes = selectedFields.map(fieldTypes(_))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
index 5cb7e90..ef1aa16 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
@@ -89,14 +89,14 @@ object TableSourceUtil {
val (physicalName, _, tpe) = resolveInputField(name, tableSource)
// validate that mapped fields are are same type
if (tpe != t) {
- throw ValidationException(s"Type $t of table field '$name' does not " +
+ throw new ValidationException(s"Type $t of table field '$name' does not " +
s"match with type $tpe of the field '$physicalName' of the TableSource return type.")
}
mappedFieldCnt += 1
}
// ensure that only one field is mapped to an atomic type
if (!tableSource.getReturnType.isInstanceOf[CompositeType[_]] && mappedFieldCnt > 1) {
- throw ValidationException(
+ throw new ValidationException(
s"More than one table field matched to atomic input type ${tableSource.getReturnType}.")
}
@@ -105,7 +105,8 @@ object TableSourceUtil {
case r: DefinedRowtimeAttributes =>
val descriptors = r.getRowtimeAttributeDescriptors
if (descriptors.size() > 1) {
- throw ValidationException("Currently, only a single rowtime attribute is supported. " +
+ throw new ValidationException(
+ "Currently, only a single rowtime attribute is supported. " +
s"Please remove all but one RowtimeAttributeDescriptor.")
} else if (descriptors.size() == 1) {
val descriptor = descriptors.get(0)
@@ -113,12 +114,12 @@ object TableSourceUtil {
val rowtimeIdx = schema.getColumnNames.indexOf(rowtimeAttribute)
// ensure that field exists
if (rowtimeIdx < 0) {
- throw ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
+ throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$rowtimeAttribute' but field '$rowtimeAttribute' does not exist in table.")
}
// ensure that field is of type TIMESTAMP
if (schema.getTypes(rowtimeIdx) != Types.SQL_TIMESTAMP) {
- throw ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
+ throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$rowtimeAttribute' but field '$rowtimeAttribute' is not of type TIMESTAMP.")
}
// look up extractor input fields in return type
@@ -137,12 +138,12 @@ object TableSourceUtil {
val proctimeIdx = schema.getColumnNames.indexOf(proctimeAttribute)
// ensure that field exists
if (proctimeIdx < 0) {
- throw ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
+ throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$proctimeAttribute' but field '$proctimeAttribute' does not exist in table.")
}
// ensure that field is of type TIMESTAMP
if (schema.getTypes(proctimeIdx) != Types.SQL_TIMESTAMP) {
- throw ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
+ throw new ValidationException(s"Found a RowtimeAttributeDescriptor for field " +
s"'$proctimeAttribute' but field '$proctimeAttribute' is not of type TIMESTAMP.")
}
case _ => // nothing to validate
@@ -224,7 +225,7 @@ object TableSourceUtil {
val (physicalName, idx, tpe) = resolveInputField(name, tableSource)
// validate that mapped fields are are same type
if (tpe != t) {
- throw ValidationException(s"Type $t of table field '$name' does not " +
+ throw new ValidationException(s"Type $t of table field '$name' does not " +
s"match with type $tpe of the field '$physicalName' of the TableSource return type.")
}
idx
@@ -232,7 +233,7 @@ object TableSourceUtil {
// ensure that only one field is mapped to an atomic type
if (!inputType.isInstanceOf[CompositeType[_]] && mapping.count(_ >= 0) > 1) {
- throw ValidationException(
+ throw new ValidationException(
s"More than one table field matched to atomic input type $inputType.")
}
@@ -302,7 +303,7 @@ object TableSourceUtil {
if (descriptors.size() == 0) {
None
} else if (descriptors.size > 1) {
- throw ValidationException("Table with has more than a single rowtime attribute..")
+ throw new ValidationException("Table with has more than a single rowtime attribute.")
} else {
// exactly one rowtime attribute descriptor
if (selectedFields.isEmpty) {
@@ -418,7 +419,7 @@ object TableSourceUtil {
val accessedFields = if (rowtimeAttributeDescriptor.isDefined) {
rowtimeAttributeDescriptor.get.getTimestampExtractor.getArgumentFields
} else {
- throw TableException("Computed field mapping includes a rowtime marker but the " +
+ throw new TableException("Computed field mapping includes a rowtime marker but the " +
"TableSource does not provide a RowtimeAttributeDescriptor. " +
"This is a bug and should be reported.")
}
@@ -471,7 +472,7 @@ object TableSourceUtil {
// get and check field index
val idx = c.getFieldIndex(fieldName)
if (idx < 0) {
- throw ValidationException(failMsg)
+ throw new ValidationException(failMsg)
}
// return field name, index, and field type
(fieldName, idx, c.getTypeAt(idx))
@@ -487,7 +488,7 @@ object TableSourceUtil {
// resolve field name in field mapping
val resolvedFieldName = d.getFieldMapping.get(fieldName)
if (resolvedFieldName == null) {
- throw ValidationException(
+ throw new ValidationException(
s"Field '$fieldName' could not be resolved by the field mapping.")
}
// look up resolved field in return type
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala
index 979f869..2468b1a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/ExistingField.scala
@@ -42,7 +42,7 @@ final class ExistingField(val field: String) extends TimestampExtractor {
case Types.SQL_TIMESTAMP => // OK
case Types.STRING => // OK
case _: TypeInformation[_] =>
- throw ValidationException(
+ throw new ValidationException(
s"Field '$field' must be of type Long or Timestamp or String but is of type $fieldType.")
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
index ff7fbf5..efcd006 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/validate/FunctionCatalog.scala
@@ -64,7 +64,7 @@ class FunctionCatalog {
*/
def lookupFunction(name: String, children: Seq[Expression]): Expression = {
val funcClass = functionBuilders
- .getOrElse(name.toLowerCase, throw ValidationException(s"Undefined function: $name"))
+ .getOrElse(name.toLowerCase, throw new ValidationException(s"Undefined function: $name"))
// Instantiate a function using the provided `children`
funcClass match {
@@ -73,7 +73,7 @@ class FunctionCatalog {
case sf if classOf[ScalarFunction].isAssignableFrom(sf) =>
val scalarSqlFunction = sqlFunctions
.find(f => f.getName.equalsIgnoreCase(name) && f.isInstanceOf[ScalarSqlFunction])
- .getOrElse(throw ValidationException(s"Undefined scalar function: $name"))
+ .getOrElse(throw new ValidationException(s"Undefined scalar function: $name"))
.asInstanceOf[ScalarSqlFunction]
ScalarFunctionCall(scalarSqlFunction.getScalarFunction, children)
@@ -81,7 +81,7 @@ class FunctionCatalog {
case tf if classOf[TableFunction[_]].isAssignableFrom(tf) =>
val tableSqlFunction = sqlFunctions
.find(f => f.getName.equalsIgnoreCase(name) && f.isInstanceOf[TableSqlFunction])
- .getOrElse(throw ValidationException(s"Undefined table function: $name"))
+ .getOrElse(throw new ValidationException(s"Undefined table function: $name"))
.asInstanceOf[TableSqlFunction]
val typeInfo = tableSqlFunction.getRowTypeInfo
val function = tableSqlFunction.getTableFunction
@@ -91,7 +91,7 @@ class FunctionCatalog {
case af if classOf[AggregateFunction[_, _]].isAssignableFrom(af) =>
val aggregateFunction = sqlFunctions
.find(f => f.getName.equalsIgnoreCase(name) && f.isInstanceOf[AggSqlFunction])
- .getOrElse(throw ValidationException(s"Undefined table function: $name"))
+ .getOrElse(throw new ValidationException(s"Undefined table function: $name"))
.asInstanceOf[AggSqlFunction]
val function = aggregateFunction.getFunction
val returnType = aggregateFunction.returnType
@@ -121,16 +121,16 @@ class FunctionCatalog {
case Success(ctor) =>
Try(ctor.newInstance(children: _*).asInstanceOf[Expression]) match {
case Success(expr) => expr
- case Failure(exception) => throw ValidationException(exception.getMessage)
+ case Failure(exception) => throw new ValidationException(exception.getMessage)
}
case Failure(_) =>
- throw ValidationException(
+ throw new ValidationException(
s"Invalid number of arguments for function $funcClass")
}
}
}
case _ =>
- throw ValidationException("Unsupported function.")
+ throw new ValidationException("Unsupported function.")
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
index 199d416..02ff5ce 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
@@ -124,7 +124,7 @@ object RowtimeTest {
argumentFieldTypes(0) match {
case Types.SQL_TIMESTAMP =>
case _ =>
- throw ValidationException(
+ throw new ValidationException(
s"Field 'ts' must be of type Timestamp but is of type ${argumentFieldTypes(0)}.")
}
}
[flink] 06/07: [FLINK-10687] [table] Move format factories to
flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 95b40868ef19a38353ecbb84ea3946e90e2e38e3
Author: Timo Walther <tw...@apache.org>
AuthorDate: Sat Oct 27 07:45:41 2018 +0200
[FLINK-10687] [table] Move format factories to flink-table-common
---
.../factories/DeserializationSchemaFactory.java} | 32 ++++----
.../factories/SerializationSchemaFactory.java} | 32 ++++----
.../apache/flink/table/factories/TableFactory.java | 83 +++++++++++++++++++++
.../flink/table/factories/TableFormatFactory.java | 65 +++++++++++++++++
.../flink/table/factories/TableFactory.scala | 85 ----------------------
.../flink/table/factories/TableFormatFactory.scala | 65 -----------------
6 files changed, 180 insertions(+), 182 deletions(-)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/DeserializationSchemaFactory.scala b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
similarity index 56%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/DeserializationSchemaFactory.scala
rename to flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
index 92aa8c9..12af6ce 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/DeserializationSchemaFactory.scala
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/DeserializationSchemaFactory.java
@@ -16,26 +16,26 @@
* limitations under the License.
*/
-package org.apache.flink.table.factories
+package org.apache.flink.table.factories;
-import java.util
+import org.apache.flink.api.common.serialization.DeserializationSchema;
-import org.apache.flink.api.common.serialization.DeserializationSchema
+import java.util.Map;
/**
- * Factory for creating configured instances of [[DeserializationSchema]].
- *
- * @tparam T record type that the format produces or consumes
- */
-trait DeserializationSchemaFactory[T] extends TableFormatFactory[T] {
+ * Factory for creating configured instances of {@link DeserializationSchema}.
+ *
+ * @param <T> record type that the format produces or consumes.
+ */
+public interface DeserializationSchemaFactory<T> extends TableFormatFactory<T> {
- /**
- * Creates and configures a [[DeserializationSchema]] using the given properties.
- *
- * @param properties normalized properties describing the format
- * @return the configured serialization schema or null if the factory cannot provide an
- * instance of this class
- */
- def createDeserializationSchema(properties: util.Map[String, String]): DeserializationSchema[T]
+ /**
+ * Creates and configures a {@link DeserializationSchema} using the given properties.
+ *
+ * @param properties normalized properties describing the format
+ * @return the configured serialization schema or null if the factory cannot provide an
+ * instance of this class
+ */
+ DeserializationSchema<T> createDeserializationSchema(Map<String, String> properties);
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/SerializationSchemaFactory.scala b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
similarity index 56%
rename from flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/SerializationSchemaFactory.scala
rename to flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
index 1f6b6bf..20c8296 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/SerializationSchemaFactory.scala
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/SerializationSchemaFactory.java
@@ -16,26 +16,26 @@
* limitations under the License.
*/
-package org.apache.flink.table.factories
+package org.apache.flink.table.factories;
-import java.util
+import org.apache.flink.api.common.serialization.SerializationSchema;
-import org.apache.flink.api.common.serialization.SerializationSchema
+import java.util.Map;
/**
- * Factory for creating configured instances of [[SerializationSchema]].
- *
- * @tparam T record type that the format produces or consumes
- */
-trait SerializationSchemaFactory[T] extends TableFormatFactory[T] {
+ * Factory for creating configured instances of {@link SerializationSchema}.
+ *
+ * @param <T> record type that the format produces or consumes.
+ */
+public interface SerializationSchemaFactory<T> extends TableFormatFactory<T> {
- /**
- * Creates and configures a [[SerializationSchema]] using the given properties.
- *
- * @param properties normalized properties describing the format
- * @return the configured serialization schema or null if the factory cannot provide an
- * instance of this class
- */
- def createSerializationSchema(properties: util.Map[String, String]): SerializationSchema[T]
+ /**
+ * Creates and configures a [[SerializationSchema]] using the given properties.
+ *
+ * @param properties normalized properties describing the format
+ * @return the configured serialization schema or null if the factory cannot provide an
+ * instance of this class
+ */
+ SerializationSchema<T> createSerializationSchema(Map<String, String> properties);
}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
new file mode 100644
index 0000000..647feb5
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.factories;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A factory to create different table-related instances from string-based properties. This
+ * factory is used with Java's Service Provider Interfaces (SPI) for discovering. A factory is
+ * called with a set of normalized properties that describe the desired configuration. The factory
+ * allows for matching to the given set of properties.
+ *
+ * <p>Classes that implement this interface can be added to the
+ * "META_INF/services/org.apache.flink.table.factories.TableFactory" file of a JAR file in
+ * the current classpath to be found.
+ *
+ * @see TableFormatFactory
+ */
+public interface TableFactory {
+
+ /**
+ * Specifies the context that this factory has been implemented for. The framework guarantees to
+ * only match for this factory if the specified set of properties and values are met.
+ *
+ * <p>Typical properties might be:
+ * - connector.type
+ * - format.type
+ *
+ * <p>Specified property versions allow the framework to provide backwards compatible properties
+ * in case of string format changes:
+ * - connector.property-version
+ * - format.property-version
+ *
+ * <p>An empty context means that the factory matches for all requests.
+ */
+ Map<String, String> requiredContext();
+
+
+ /**
+ * List of property keys that this factory can handle. This method will be used for validation.
+ * If a property is passed that this factory cannot handle, an exception will be thrown. The
+ * list must not contain the keys that are specified by the context.
+ *
+ * <p>Example properties might be:
+ * - schema.#.type
+ * - schema.#.name
+ * - connector.topic
+ * - format.line-delimiter
+ * - format.ignore-parse-errors
+ * - format.fields.#.type
+ * - format.fields.#.name
+ *
+ * <p>Note: Use "#" to denote an array of values where "#" represents one or more digits. Property
+ * versions like "format.property-version" must not be part of the supported properties.
+ *
+ * <p>In some cases it might be useful to declare wildcards "*". Wildcards can only be declared at
+ * the end of a property key.
+ *
+ * <p>For example, if an arbitrary format should be supported:
+ * - format.*
+ *
+ * <p>Note: Wildcards should be used with caution as they might swallow unsupported properties
+ * and thus might lead to undesired behavior.
+ */
+ List<String> supportedProperties();
+}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
new file mode 100644
index 0000000..da298f2
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/factories/TableFormatFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.factories;
+
+import java.util.List;
+
+/**
+ * A factory to create configured table format instances based on string-based properties. See
+ * also {@link TableFactory} for more information.
+ *
+ * @see DeserializationSchemaFactory
+ * @see SerializationSchemaFactory
+ *
+ * @param <T> record type that the format produces or consumes.
+ */
+public interface TableFormatFactory<T> extends TableFactory {
+
+ /**
+ * Flag to indicate if the given format supports deriving information from a schema. If the
+ * format can handle schema information, those properties must be added to the list of
+ * supported properties.
+ */
+ boolean supportsSchemaDerivation();
+
+ /**
+ * List of format property keys that this factory can handle. This method will be used for
+ * validation. If a property is passed that this factory cannot handle, an exception will be
+ * thrown. The list must not contain the keys that are specified by the context.
+ *
+ * <p>Example format properties might be:
+ * - format.line-delimiter
+ * - format.ignore-parse-errors
+ * - format.fields.#.type
+ * - format.fields.#.name
+ *
+ * <p>If schema derivation is enabled, the list must include schema properties:
+ * - schema.#.name
+ * - schema.#.type
+ *
+ * <p>Note: All supported format properties must be prefixed with "format.". If schema derivation is
+ * enabled, also properties with "schema." prefix can be used.
+ *
+ * <p>Use "#" to denote an array of values where "#" represents one or more digits. Property
+ * versions like "format.property-version" must not be part of the supported properties.
+ *
+ * <p>See also {@link TableFactory#supportedProperties()} for more information.
+ */
+ List<String> supportedProperties();
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactory.scala
deleted file mode 100644
index a5dcc60..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactory.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.factories
-
-import java.util
-
-/**
- * A factory to create different table-related instances from string-based properties. This
- * factory is used with Java's Service Provider Interfaces (SPI) for discovering. A factory is
- * called with a set of normalized properties that describe the desired configuration. The factory
- * allows for matching to the given set of properties.
- *
- * Classes that implement this interface can be added to the
- * "META_INF/services/org.apache.flink.table.factories.TableFactory" file of a JAR file in
- * the current classpath to be found.
- *
- * @see [[BatchTableSourceFactory]]
- * @see [[BatchTableSinkFactory]]
- * @see [[StreamTableSourceFactory]]
- * @see [[StreamTableSinkFactory]]
- * @see [[TableFormatFactory]]
- */
-trait TableFactory {
-
- /**
- * Specifies the context that this factory has been implemented for. The framework guarantees to
- * only match for this factory if the specified set of properties and values are met.
- *
- * Typical properties might be:
- * - connector.type
- * - format.type
- *
- * Specified property versions allow the framework to provide backwards compatible properties
- * in case of string format changes:
- * - connector.property-version
- * - format.property-version
- *
- * An empty context means that the factory matches for all requests.
- */
- def requiredContext(): util.Map[String, String]
-
- /**
- * List of property keys that this factory can handle. This method will be used for validation.
- * If a property is passed that this factory cannot handle, an exception will be thrown. The
- * list must not contain the keys that are specified by the context.
- *
- * Example properties might be:
- * - schema.#.type
- * - schema.#.name
- * - connector.topic
- * - format.line-delimiter
- * - format.ignore-parse-errors
- * - format.fields.#.type
- * - format.fields.#.name
- *
- * Note: Use "#" to denote an array of values where "#" represents one or more digits. Property
- * versions like "format.property-version" must not be part of the supported properties.
- *
- * In some cases it might be useful to declare wildcards "*". Wildcards can only be declared at
- * the end of a property key.
- *
- * For example, if an arbitrary format should be supported:
- * - format.*
- *
- * Note: Wildcards should be used with caution as they might swallow unsupported properties
- * and thus might lead to undesired behavior.
- */
- def supportedProperties(): util.List[String]
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFormatFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFormatFactory.scala
deleted file mode 100644
index 9e42a15..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFormatFactory.scala
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.factories
-
-import java.util
-
-/**
- * A factory to create configured table format instances based on string-based properties. See
- * also [[TableFactory]] for more information.
- *
- * @tparam T record type that the format produces or consumes
- * @see [[DeserializationSchemaFactory]]
- * @see [[SerializationSchemaFactory]]
- */
-trait TableFormatFactory[T] extends TableFactory {
-
- /**
- * Flag to indicate if the given format supports deriving information from a schema. If the
- * format can handle schema information, those properties must be added to the list of
- * supported properties.
- */
- def supportsSchemaDerivation(): Boolean
-
- /**
- * List of format property keys that this factory can handle. This method will be used for
- * validation. If a property is passed that this factory cannot handle, an exception will be
- * thrown. The list must not contain the keys that are specified by the context.
- *
- * Example format properties might be:
- * - format.line-delimiter
- * - format.ignore-parse-errors
- * - format.fields.#.type
- * - format.fields.#.name
- *
- * If schema derivation is enabled, the list must include schema properties:
- * - schema.#.name
- * - schema.#.type
- *
- * Note: All supported format properties must be prefixed with "format.". If schema derivation is
- * enabled, also properties with "schema." prefix can be used.
- *
- * Use "#" to denote an array of values where "#" represents one or more digits. Property
- * versions like "format.property-version" must not be part of the supported properties.
- *
- * @see See also [[TableFactory.supportedProperties()]] for more information.
- */
- def supportedProperties(): util.List[String]
-
-}
[flink] 05/07: [FLINK-10687] [table] Move DescriptorProperties to
flink-table-common
Posted by tw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3bfe134b7602e749074ded3826689328c29c3ab1
Author: Timo Walther <tw...@apache.org>
AuthorDate: Mon Oct 29 09:18:39 2018 +0100
[FLINK-10687] [table] Move DescriptorProperties to flink-table-common
---
.../table/descriptors/ElasticsearchValidator.java | 21 +-
...lasticsearchUpsertTableSinkFactoryTestBase.java | 4 +-
.../flink/table/descriptors/KafkaValidator.java | 28 +-
.../kafka/KafkaTableSourceSinkFactoryTestBase.java | 6 +-
.../flink/table/descriptors/JsonValidator.java | 2 +-
.../table/descriptors/DescriptorProperties.java | 1342 ++++++++++++++++
.../org/apache/flink/table/api/exceptions.scala | 6 +-
.../flink/table/catalog/ExternalCatalogTable.scala | 9 +-
.../table/descriptors/ClassInstanceValidator.scala | 2 +-
.../descriptors/ConnectorDescriptorValidator.scala | 4 +-
.../org/apache/flink/table/descriptors/Csv.scala | 23 +-
.../flink/table/descriptors/CsvValidator.scala | 16 +-
.../table/descriptors/DescriptorProperties.scala | 1601 --------------------
.../table/descriptors/FileSystemValidator.scala | 4 +-
.../descriptors/FormatDescriptorValidator.scala | 4 +-
.../descriptors/FunctionDescriptorValidator.scala | 4 +-
.../table/descriptors/LiteralValueValidator.scala | 22 +-
.../table/descriptors/MetadataValidator.scala | 8 +-
.../flink/table/descriptors/RowtimeValidator.scala | 35 +-
.../apache/flink/table/descriptors/Schema.scala | 10 +-
.../flink/table/descriptors/SchemaValidator.scala | 10 +-
.../table/descriptors/StatisticsValidator.scala | 20 +-
.../StreamTableDescriptorValidator.scala | 2 +-
.../table/factories/TableFactoryService.scala | 6 +-
.../flink/table/factories/TableFactoryUtil.scala | 7 +-
.../table/sinks/CsvTableSinkFactoryBase.scala | 10 +-
.../table/sources/CsvTableSourceFactoryBase.scala | 10 +-
.../flink/table/util/JavaScalaConversionUtil.scala | 69 +
.../descriptors/DescriptorPropertiesTest.scala | 20 +-
.../table/descriptors/DescriptorTestBase.scala | 10 +-
30 files changed, 1571 insertions(+), 1744 deletions(-)
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
index 9e04f2f..1858ee1 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/main/java/org/apache/flink/table/descriptors/ElasticsearchValidator.java
@@ -24,6 +24,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
+import static org.apache.flink.table.descriptors.DescriptorProperties.noValidation;
+
/**
* The validator for {@link Elasticsearch}.
*/
@@ -79,14 +81,9 @@ public class ElasticsearchValidator extends ConnectorDescriptorValidator {
private void validateHosts(DescriptorProperties properties) {
final Map<String, Consumer<String>> hostsValidators = new HashMap<>();
- hostsValidators.put(
- CONNECTOR_HOSTS_HOSTNAME,
- (prefix) -> properties.validateString(prefix + CONNECTOR_HOSTS_HOSTNAME, false, 1));
- hostsValidators.put(
- CONNECTOR_HOSTS_PORT,
- (prefix) -> properties.validateInt(prefix + CONNECTOR_HOSTS_PORT, false, 0, 65535));
- hostsValidators.put(CONNECTOR_HOSTS_PROTOCOL,
- (prefix) -> properties.validateString(prefix + CONNECTOR_HOSTS_PROTOCOL, false, 1));
+ hostsValidators.put(CONNECTOR_HOSTS_HOSTNAME, (key) -> properties.validateString(key, false, 1));
+ hostsValidators.put(CONNECTOR_HOSTS_PORT, (key) -> properties.validateInt(key, false, 0, 65535));
+ hostsValidators.put(CONNECTOR_HOSTS_PROTOCOL, (key) -> properties.validateString(key, false, 1));
properties.validateFixedIndexedProperties(CONNECTOR_HOSTS, false, hostsValidators);
}
@@ -99,11 +96,11 @@ public class ElasticsearchValidator extends ConnectorDescriptorValidator {
private void validateFailureHandler(DescriptorProperties properties) {
final Map<String, Consumer<String>> failureHandlerValidators = new HashMap<>();
- failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_FAIL, properties.noValidation());
- failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_IGNORE, properties.noValidation());
- failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_RETRY, properties.noValidation());
+ failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_FAIL, noValidation());
+ failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_IGNORE, noValidation());
+ failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_RETRY, noValidation());
failureHandlerValidators.put(CONNECTOR_FAILURE_HANDLER_VALUE_CUSTOM,
- prefix -> properties.validateString(CONNECTOR_FAILURE_HANDLER_CLASS, false, 1));
+ key -> properties.validateString(CONNECTOR_FAILURE_HANDLER_CLASS, false, 1));
properties.validateEnum(CONNECTOR_FAILURE_HANDLER, true, failureHandlerValidators);
}
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
index 9376bfb..88aee43 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
@@ -24,7 +24,6 @@ import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTa
import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.api.Types;
-import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.descriptors.Elasticsearch;
import org.apache.flink.table.descriptors.Json;
import org.apache.flink.table.descriptors.Schema;
@@ -32,6 +31,7 @@ import org.apache.flink.table.descriptors.TestTableDescriptor;
import org.apache.flink.table.factories.StreamTableSinkFactory;
import org.apache.flink.table.factories.TableFactoryService;
import org.apache.flink.table.sinks.TableSink;
+import org.apache.flink.table.util.JavaScalaConversionUtil;
import org.apache.flink.types.Row;
import org.apache.flink.util.TestLogger;
@@ -113,7 +113,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryTestBase extends TestLo
.field(FIELD_TS, Types.SQL_TIMESTAMP()))
.inUpsertMode();
- final Map<String, String> propertiesMap = DescriptorProperties.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
.createStreamTableSink(propertiesMap);
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
index c7ddb7f..68939d2 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
+import static org.apache.flink.table.descriptors.DescriptorProperties.noValidation;
+
/**
* The validator for {@link Kafka}.
*/
@@ -84,26 +86,26 @@ public class KafkaValidator extends ConnectorDescriptorValidator {
final Map<String, Consumer<String>> specificOffsetValidators = new HashMap<>();
specificOffsetValidators.put(
CONNECTOR_SPECIFIC_OFFSETS_PARTITION,
- (prefix) -> properties.validateInt(
- prefix + CONNECTOR_SPECIFIC_OFFSETS_PARTITION,
+ (key) -> properties.validateInt(
+ key,
false,
0,
Integer.MAX_VALUE));
specificOffsetValidators.put(
CONNECTOR_SPECIFIC_OFFSETS_OFFSET,
- (prefix) -> properties.validateLong(
- prefix + CONNECTOR_SPECIFIC_OFFSETS_OFFSET,
+ (key) -> properties.validateLong(
+ key,
false,
0,
Long.MAX_VALUE));
final Map<String, Consumer<String>> startupModeValidation = new HashMap<>();
- startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS, properties.noValidation());
- startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_EARLIEST, properties.noValidation());
- startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_LATEST, properties.noValidation());
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS, noValidation());
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_EARLIEST, noValidation());
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_LATEST, noValidation());
startupModeValidation.put(
CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS,
- prefix -> properties.validateFixedIndexedProperties(CONNECTOR_SPECIFIC_OFFSETS, false, specificOffsetValidators));
+ key -> properties.validateFixedIndexedProperties(CONNECTOR_SPECIFIC_OFFSETS, false, specificOffsetValidators));
properties.validateEnum(CONNECTOR_STARTUP_MODE, true, startupModeValidation);
}
@@ -111,20 +113,20 @@ public class KafkaValidator extends ConnectorDescriptorValidator {
final Map<String, Consumer<String>> propertyValidators = new HashMap<>();
propertyValidators.put(
CONNECTOR_PROPERTIES_KEY,
- prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_KEY, false, 1));
+ key -> properties.validateString(key, false, 1));
propertyValidators.put(
CONNECTOR_PROPERTIES_VALUE,
- prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_VALUE, false, 0));
+ key -> properties.validateString(key, false, 0));
properties.validateFixedIndexedProperties(CONNECTOR_PROPERTIES, true, propertyValidators);
}
private void validateSinkPartitioner(DescriptorProperties properties) {
final Map<String, Consumer<String>> sinkPartitionerValidators = new HashMap<>();
- sinkPartitionerValidators.put(CONNECTOR_SINK_PARTITIONER_VALUE_FIXED, properties.noValidation());
- sinkPartitionerValidators.put(CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN, properties.noValidation());
+ sinkPartitionerValidators.put(CONNECTOR_SINK_PARTITIONER_VALUE_FIXED, noValidation());
+ sinkPartitionerValidators.put(CONNECTOR_SINK_PARTITIONER_VALUE_ROUND_ROBIN, noValidation());
sinkPartitionerValidators.put(
CONNECTOR_SINK_PARTITIONER_VALUE_CUSTOM,
- prefix -> properties.validateString(CONNECTOR_SINK_PARTITIONER_CLASS, false, 1));
+ key -> properties.validateString(CONNECTOR_SINK_PARTITIONER_CLASS, false, 1));
properties.validateEnum(CONNECTOR_SINK_PARTITIONER, true, sinkPartitionerValidators);
}
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
index 135e718..4beba49 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
@@ -36,7 +36,6 @@ import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartiti
import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.api.Types;
-import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.descriptors.Kafka;
import org.apache.flink.table.descriptors.Rowtime;
import org.apache.flink.table.descriptors.Schema;
@@ -53,6 +52,7 @@ import org.apache.flink.table.sources.TableSource;
import org.apache.flink.table.sources.TableSourceUtil;
import org.apache.flink.table.sources.tsextractors.ExistingField;
import org.apache.flink.table.sources.wmstrategies.AscendingTimestamps;
+import org.apache.flink.table.util.JavaScalaConversionUtil;
import org.apache.flink.types.Row;
import org.apache.flink.util.TestLogger;
@@ -165,7 +165,7 @@ public abstract class KafkaTableSourceSinkFactoryTestBase extends TestLogger {
.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
.inAppendMode();
- final Map<String, String> propertiesMap = DescriptorProperties.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
.createStreamTableSource(propertiesMap);
@@ -215,7 +215,7 @@ public abstract class KafkaTableSourceSinkFactoryTestBase extends TestLogger {
.field(EVENT_TIME, Types.SQL_TIMESTAMP()))
.inAppendMode();
- final Map<String, String> propertiesMap = DescriptorProperties.toJavaMap(testDesc);
+ final Map<String, String> propertiesMap = JavaScalaConversionUtil.toJavaMap(testDesc);
final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
.createStreamTableSink(propertiesMap);
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
index 49e1abc..af8721f 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
@@ -45,7 +45,7 @@ public class JsonValidator extends FormatDescriptorValidator {
} else if (!deriveSchema && !hasSchema && !hasSchemaString) {
throw new ValidationException("A definition of a schema or JSON schema is required.");
} else if (hasSchema) {
- properties.validateType(FORMAT_SCHEMA, true, false);
+ properties.validateType(FORMAT_SCHEMA, false, true);
} else if (hasSchemaString) {
properties.validateString(FORMAT_JSON_SCHEMA, false, 1);
}
diff --git a/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java
new file mode 100644
index 0000000..0d2f835
--- /dev/null
+++ b/flink-libraries/flink-table-common/src/main/java/org/apache/flink/table/descriptors/DescriptorProperties.java
@@ -0,0 +1,1342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.configuration.MemorySize;
+import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.utils.EncodingUtils;
+import org.apache.flink.table.utils.TypeStringUtils;
+import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
+
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Utility class for having a unified string-based representation of Table API related classes
+ * such as TableSchema, TypeInformation, etc.
+ *
+ * <p>Note to implementers: Please try to reuse key names as much as possible. Key-names
+ * should be hierarchical and lower case. Use "-" instead of dots or camel case.
+ * E.g., connector.schema.start-from = from-earliest. Try not to use the higher level in a
+ * key-name. E.g., instead of connector.kafka.kafka-version use connector.kafka.version.
+ *
+ * <p>Properties with key normalization enabled contain only lower-case keys.
+ */
+@Internal
+public class DescriptorProperties {
+
+ public static final String TABLE_SCHEMA_NAME = "name";
+
+ public static final String TABLE_SCHEMA_TYPE = "type";
+
+ private static final Consumer<String> EMPTY_CONSUMER = (value) -> {};
+
+ private final boolean normalizeKeys;
+
+ private final Map<String, String> properties;
+
+ public DescriptorProperties(boolean normalizeKeys) {
+ this.properties = new HashMap<>();
+ this.normalizeKeys = normalizeKeys;
+ }
+
+ public DescriptorProperties() {
+ this(true);
+ }
+
+ /**
+ * Adds a set of properties.
+ */
+ public void putProperties(Map<String, String> properties) {
+ for (Map.Entry<String, String> property : properties.entrySet()) {
+ put(property.getKey(), property.getValue());
+ }
+ }
+
+ /**
+ * Adds a set of descriptor properties.
+ */
+ public void putProperties(DescriptorProperties otherProperties) {
+ for (Map.Entry<String, String> otherProperty : otherProperties.properties.entrySet()) {
+ put(otherProperty.getKey(), otherProperty.getValue());
+ }
+ }
+
+ /**
+ * Adds a class under the given key.
+ */
+ public void putClass(String key, Class<?> clazz) {
+ checkNotNull(key);
+ checkNotNull(clazz);
+ final String error = InstantiationUtil.checkForInstantiationError(clazz);
+ if (error != null) {
+ throw new ValidationException("Class '" + clazz.getName() + "' is not supported: " + error);
+ }
+ put(key, clazz.getName());
+ }
+
+ /**
+ * Adds a string under the given key.
+ */
+ public void putString(String key, String str) {
+ checkNotNull(key);
+ checkNotNull(str);
+ put(key, str);
+ }
+
+ /**
+ * Adds a boolean under the given key.
+ */
+ public void putBoolean(String key, boolean b) {
+ checkNotNull(key);
+ put(key, Boolean.toString(b));
+ }
+
+ /**
+ * Adds a long under the given key.
+ */
+ public void putLong(String key, long l) {
+ checkNotNull(key);
+ put(key, Long.toString(l));
+ }
+
+ /**
+ * Adds an integer under the given key.
+ */
+ public void putInt(String key, int i) {
+ checkNotNull(key);
+ put(key, Integer.toString(i));
+ }
+
+ /**
+ * Adds a character under the given key.
+ */
+ public void putCharacter(String key, char c) {
+ checkNotNull(key);
+ put(key, Character.toString(c));
+ }
+
+ /**
+ * Adds a table schema under the given key.
+ */
+ public void putTableSchema(String key, TableSchema schema) {
+ checkNotNull(key);
+ checkNotNull(schema);
+
+ final String[] fieldNames = schema.getFieldNames();
+ final TypeInformation<?>[] fieldTypes = schema.getFieldTypes();
+
+ final List<List<String>> values = new ArrayList<>();
+ for (int i = 0; i < schema.getFieldCount(); i++) {
+ values.add(Arrays.asList(fieldNames[i], TypeStringUtils.writeTypeInfo(fieldTypes[i])));
+ }
+
+ putIndexedFixedProperties(
+ key,
+ Arrays.asList(TABLE_SCHEMA_NAME, TABLE_SCHEMA_TYPE),
+ values);
+ }
+
+ /**
+ * Adds a Flink {@link MemorySize} under the given key.
+ */
+ public void putMemorySize(String key, MemorySize size) {
+ checkNotNull(key);
+ checkNotNull(size);
+ put(key, size.toString());
+ }
+
+ /**
+ * Adds an indexed sequence of properties (with sub-properties) under a common key.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ * </pre>
+ *
+ * <p>The arity of each subKeyValues must match the arity of propertyKeys.
+ */
+ public void putIndexedFixedProperties(String key, List<String> subKeys, List<List<String>> subKeyValues) {
+ checkNotNull(key);
+ checkNotNull(subKeys);
+ checkNotNull(subKeyValues);
+ for (int idx = 0; idx < subKeyValues.size(); idx++) {
+ final List<String> values = subKeyValues.get(idx);
+ if (values == null || values.size() != subKeys.size()) {
+ throw new ValidationException("Values must have same arity as keys.");
+ }
+ for (int keyIdx = 0; keyIdx < values.size(); keyIdx++) {
+ put(key + '.' + idx + '.' + subKeys.get(keyIdx), values.get(keyIdx));
+ }
+ }
+ }
+
+ /**
+ * Adds an indexed mapping of properties under a common key.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.name = test2
+ * </pre>
+ *
+ * <p>The arity of the subKeyValues can differ.
+ */
+ public void putIndexedVariableProperties(String key, List<Map<String, String>> subKeyValues) {
+ checkNotNull(key);
+ checkNotNull(subKeyValues);
+ for (int idx = 0; idx < subKeyValues.size(); idx++) {
+ final Map<String, String> values = subKeyValues.get(idx);
+ for (Map.Entry<String, String> value : values.entrySet()) {
+ put(key + '.' + idx + '.' + value.getKey(), value.getValue());
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Returns a string value under the given key if it exists.
+ */
+ public Optional<String> getOptionalString(String key) {
+ return optionalGet(key);
+ }
+
+ /**
+ * Returns a string value under the given existing key.
+ */
+ public String getString(String key) {
+ return getOptionalString(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a character value under the given key if it exists.
+ */
+ public Optional<Character> getOptionalCharacter(String key) {
+ return optionalGet(key).map((c) -> {
+ if (c.length() != 1) {
+ throw new ValidationException("The value of '" + key + "' must only contain one character.");
+ }
+ return c.charAt(0);
+ });
+ }
+
+ /**
+ * Returns a character value under the given existing key.
+ */
+ public Character getCharacter(String key) {
+ return getOptionalCharacter(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a class value under the given key if it exists.
+ */
+ @SuppressWarnings("unchecked")
+ public <T> Optional<Class<T>> getOptionalClass(String key, Class<T> superClass) {
+ return optionalGet(key).map((name) -> {
+ final Class<?> clazz;
+ try {
+ clazz = Class.forName(name, true, Thread.currentThread().getContextClassLoader());
+ if (!superClass.isAssignableFrom(clazz)) {
+ throw new ValidationException(
+ "Class '" + name + "' does not extend from the required class '" +
+ superClass.getName() + "' for key '" + key + "'.");
+ }
+ return (Class<T>) clazz;
+ } catch (Exception e) {
+ throw new ValidationException("Could not get class '" + name + "' for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a class value under the given existing key.
+ */
+ public <T> Class<T> getClass(String key, Class<T> superClass) {
+ return getOptionalClass(key, superClass).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a big decimal value under the given key if it exists.
+ */
+ public Optional<BigDecimal> getOptionalBigDecimal(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return new BigDecimal(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid decimal value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a big decimal value under the given existing key.
+ */
+ public BigDecimal getBigDecimal(String key) {
+ return getOptionalBigDecimal(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a boolean value under the given key if it exists.
+ */
+ public Optional<Boolean> getOptionalBoolean(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Boolean.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid boolean value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a boolean value under the given existing key.
+ */
+ public boolean getBoolean(String key) {
+ return getOptionalBoolean(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a byte value under the given key if it exists.
+ */
+ public Optional<Byte> getOptionalByte(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Byte.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid byte value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a byte value under the given existing key.
+ */
+ public byte getByte(String key) {
+ return getOptionalByte(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a double value under the given key if it exists.
+ */
+ public Optional<Double> getOptionalDouble(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Double.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid double value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a double value under the given existing key.
+ */
+ public double getDouble(String key) {
+ return getOptionalDouble(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a float value under the given key if it exists.
+ */
+ public Optional<Float> getOptionalFloat(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Float.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid float value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a float value under the given given existing key.
+ */
+ public float getFloat(String key) {
+ return getOptionalFloat(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns an integer value under the given key if it exists.
+ */
+ public Optional<Integer> getOptionalInt(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Integer.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid integer value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns an integer value under the given existing key.
+ */
+ public int getInt(String key) {
+ return getOptionalInt(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a long value under the given key if it exists.
+ */
+ public Optional<Long> getOptionalLong(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Long.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid long value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a long value under the given existing key.
+ */
+ public long getLong(String key) {
+ return getOptionalLong(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a short value under the given key if it exists.
+ */
+ public Optional<Short> getOptionalShort(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return Short.valueOf(value);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid short value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a short value under the given existing key.
+ */
+ public short getShort(String key) {
+ return getOptionalShort(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns the type information under the given key if it exists.
+ */
+ public Optional<TypeInformation<?>> getOptionalType(String key) {
+ return optionalGet(key).map(TypeStringUtils::readTypeInfo);
+ }
+
+ /**
+ * Returns the type information under the given existing key.
+ */
+ public TypeInformation<?> getType(String key) {
+ return getOptionalType(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a table schema under the given key if it exists.
+ */
+ public Optional<TableSchema> getOptionalTableSchema(String key) {
+ // filter for number of fields
+ final int fieldCount = properties.keySet().stream()
+ .filter((k) -> k.startsWith(key) && k.endsWith('.' + TABLE_SCHEMA_NAME))
+ .mapToInt((k) -> 1)
+ .sum();
+
+ if (fieldCount == 0) {
+ return Optional.empty();
+ }
+
+ // validate fields and build schema
+ final TableSchema.Builder schemaBuilder = TableSchema.builder();
+ for (int i = 0; i < fieldCount; i++) {
+ final String nameKey = key + '.' + i + '.' + TABLE_SCHEMA_NAME;
+ final String typeKey = key + '.' + i + '.' + TABLE_SCHEMA_TYPE;
+
+ final String name = optionalGet(nameKey).orElseThrow(exceptionSupplier(nameKey));
+
+ final TypeInformation<?> type = optionalGet(typeKey)
+ .map(TypeStringUtils::readTypeInfo)
+ .orElseThrow(exceptionSupplier(typeKey));
+
+ schemaBuilder.field(name, type);
+ }
+ return Optional.of(schemaBuilder.build());
+ }
+
+ /**
+ * Returns a table schema under the given existing key.
+ */
+ public TableSchema getTableSchema(String key) {
+ return getOptionalTableSchema(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns a Flink {@link MemorySize} under the given key if it exists.
+ */
+ public Optional<MemorySize> getOptionalMemorySize(String key) {
+ return optionalGet(key).map((value) -> {
+ try {
+ return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES);
+ } catch (Exception e) {
+ throw new ValidationException("Invalid memory size value for key '" + key + "'.", e);
+ }
+ });
+ }
+
+ /**
+ * Returns a Flink {@link MemorySize} under the given existing key.
+ */
+ public MemorySize getMemorySize(String key) {
+ return getOptionalMemorySize(key).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns the property keys of fixed indexed properties.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ * </pre>
+ *
+ * <p>getFixedIndexedProperties("schema.fields", List("type", "name")) leads to:
+ *
+ * <pre>
+ * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
+ * 1: Map("type" -> "schema.fields.1.type", "name" -> "schema.fields.1.name")
+ * </pre>
+ */
+ public List<Map<String, String>> getFixedIndexedProperties(String key, List<String> subKeys) {
+ // determine max index
+ final int maxIndex = extractMaxIndex(key, "\\.(.*)");
+
+ // validate and create result
+ final List<Map<String, String>> list = new ArrayList<>();
+ for (int i = 0; i <= maxIndex; i++) {
+ final Map<String, String> map = new HashMap<>();
+
+ for (String subKey : subKeys) {
+ final String fullKey = key + '.' + i + '.' + subKey;
+ // check for existence of full key
+ if (!containsKey(fullKey)) {
+ throw exceptionSupplier(fullKey).get();
+ }
+ map.put(subKey, fullKey);
+ }
+
+ list.add(map);
+ }
+ return list;
+ }
+
+ /**
+ * Returns the property keys of variable indexed properties.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG
+ * </pre>
+ *
+ * <p>getFixedIndexedProperties("schema.fields", List("type")) leads to:
+ *
+ * <pre>
+ * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
+ * 1: Map("type" -> "schema.fields.1.type")
+ * </pre>
+ */
+ public List<Map<String, String>> getVariableIndexedProperties(String key, List<String> requiredSubKeys) {
+
+ // determine max index
+ final int maxIndex = extractMaxIndex(key, "(\\.)?(.*)");
+
+ // determine optional properties
+ final String escapedKey = Pattern.quote(key);
+ final Pattern pattern = Pattern.compile(escapedKey + "\\.(\\d+)(\\.)?(.*)");
+ final Set<String> optionalSubKeys = properties.keySet().stream()
+ .flatMap((k) -> {
+ final Matcher matcher = pattern.matcher(k);
+ if (matcher.find()) {
+ return Stream.of(matcher.group(3));
+ }
+ return Stream.empty();
+ })
+ .filter((k) -> k.length() > 0)
+ .collect(Collectors.toSet());
+
+ // validate and create result
+ final List<Map<String, String>> list = new ArrayList<>();
+ for (int i = 0; i <= maxIndex; i++) {
+ final Map<String, String> map = new HashMap<>();
+
+ // check and add required keys
+ for (String subKey : requiredSubKeys) {
+ final String fullKey = key + '.' + i + '.' + subKey;
+ // check for existence of full key
+ if (!containsKey(fullKey)) {
+ throw exceptionSupplier(fullKey).get();
+ }
+ map.put(subKey, fullKey);
+ }
+
+ // add optional keys
+ for (String subKey : optionalSubKeys) {
+ final String fullKey = key + '.' + i + '.' + subKey;
+ optionalGet(fullKey).ifPresent(value -> map.put(subKey, fullKey));
+ }
+
+ list.add(map);
+ }
+ return list;
+ }
+
+ /**
+ * Returns all properties under a given key that contains an index in between.
+ *
+ * <p>E.g. rowtime.0.name -> returns all rowtime.#.name properties
+ */
+ public Map<String, String> getIndexedProperty(String key, String subKey) {
+ final String escapedKey = Pattern.quote(key);
+ final String escapedSubKey = Pattern.quote(subKey);
+ return properties.entrySet().stream()
+ .filter(entry -> entry.getKey().matches(escapedKey + "\\.\\d+\\." + escapedSubKey))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ /**
+ * Returns all array elements under a given key if it exists.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * primary-key.0 = field1
+ * primary-key.1 = field2
+ * </pre>
+ *
+ * <p>leads to: List(field1, field2)
+ *
+ * <p>or:
+ *
+ * <pre>
+ * primary-key = field1
+ * </pre>
+ *
+ * <p>leads to: List(field1)
+ *
+ * <p>The key mapper gets the key of the current value e.g. "primary-key.1".
+ */
+ public <E> Optional<List<E>> getOptionalArray(String key, Function<String, E> keyMapper) {
+ // determine max index
+ final int maxIndex = extractMaxIndex(key, "");
+
+ if (maxIndex < 0) {
+ // check for a single element array
+ if (containsKey(key)) {
+ return Optional.of(Collections.singletonList(keyMapper.apply(key)));
+ } else {
+ return Optional.empty();
+ }
+ } else {
+ final List<E> list = new ArrayList<>();
+ for (int i = 0; i < maxIndex + 1; i++) {
+ final String fullKey = key + '.' + i;
+ final E value = keyMapper.apply(fullKey);
+ list.add(value);
+ }
+ return Optional.of(list);
+ }
+ }
+
+ /**
+ * Returns all array elements under a given existing key.
+ */
+ public <E> List<E> getArray(String key, Function<String, E> keyMapper) {
+ return getOptionalArray(key, keyMapper).orElseThrow(exceptionSupplier(key));
+ }
+
+ /**
+ * Returns if a value under key is exactly equal to the given value.
+ */
+ public boolean isValue(String key, String value) {
+ return optionalGet(key).orElseThrow(exceptionSupplier(key)).equals(value);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Validates a string property.
+ */
+ public void validateString(String key, boolean isOptional) {
+ validateString(key, isOptional, 0, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates a string property. The boundaries are inclusive.
+ */
+ public void validateString(String key, boolean isOptional, int minLen) {
+ validateString(key, isOptional, minLen, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates a string property. The boundaries are inclusive.
+ */
+ public void validateString(String key, boolean isOptional, int minLen, int maxLen) {
+ validateOptional(
+ key,
+ isOptional,
+ (value) -> {
+ final int length = value.length();
+ if (length < minLen || length > maxLen) {
+ throw new ValidationException(
+ "Property '" + key + "' must have a length between " + minLen + " and " + maxLen + " but was: " + value);
+ }
+ });
+ }
+
+ /**
+ * Validates an integer property.
+ */
+ public void validateInt(String key, boolean isOptional) {
+ validateInt(key, isOptional, Integer.MIN_VALUE, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates an integer property. The boundaries are inclusive.
+ */
+ public void validateInt(String key, boolean isOptional, int min) {
+ validateInt(key, isOptional, min, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates an integer property. The boundaries are inclusive.
+ */
+ public void validateInt(String key, boolean isOptional, int min, int max) {
+ validateComparable(key, isOptional, min, max, "integer", Integer::valueOf);
+ }
+
+ /**
+ * Validates an long property.
+ */
+ public void validateLong(String key, boolean isOptional) {
+ validateLong(key, isOptional, Long.MIN_VALUE, Long.MAX_VALUE);
+ }
+
+ /**
+ * Validates an long property. The boundaries are inclusive.
+ */
+ public void validateLong(String key, boolean isOptional, long min) {
+ validateLong(key, isOptional, min, Long.MAX_VALUE);
+ }
+
+ /**
+ * Validates an long property. The boundaries are inclusive.
+ */
+ public void validateLong(String key, boolean isOptional, long min, long max) {
+ validateComparable(key, isOptional, min, max, "long", Long::valueOf);
+ }
+
+ /**
+ * Validates that a certain value is present under the given key.
+ */
+ public void validateValue(String key, String value, boolean isOptional) {
+ validateOptional(
+ key,
+ isOptional,
+ (v) -> {
+ if (!v.equals(value)) {
+ throw new ValidationException(
+ "Could not find required value '" + value + "' for property '" + key + "'.");
+ }
+ });
+ }
+
+ /**
+ * Validates that a boolean value is present under the given key.
+ */
+ public void validateBoolean(String key, boolean isOptional) {
+ validateOptional(
+ key,
+ isOptional,
+ (value) -> {
+ if (!value.equalsIgnoreCase("true") && !value.equalsIgnoreCase("false")) {
+ throw new ValidationException(
+ "Property '" + key + "' must be a boolean value (true/false) but was: " + value);
+ }
+ });
+ }
+
+ /**
+ * Validates a double property.
+ */
+ public void validateDouble(String key, boolean isOptional) {
+ validateDouble(key, isOptional, Double.MIN_VALUE, Double.MAX_VALUE);
+ }
+
+ /**
+ * Validates a double property. The boundaries are inclusive.
+ */
+ public void validateDouble(String key, boolean isOptional, double min) {
+ validateDouble(key, isOptional, min, Double.MAX_VALUE);
+ }
+
+ /**
+ * Validates a double property. The boundaries are inclusive.
+ */
+ public void validateDouble(String key, boolean isOptional, double min, double max) {
+ validateComparable(key, isOptional, min, max, "double", Double::valueOf);
+ }
+
+ /**
+ * Validates a big decimal property.
+ */
+ public void validateBigDecimal(String key, boolean isOptional) {
+ validateOptional(
+ key,
+ isOptional,
+ (value) -> {
+ try {
+ new BigDecimal(value);
+ } catch (Exception e) {
+ throw new ValidationException(
+ "Property '" + key + "' must be a big decimal value but was: " + value);
+ }
+ });
+ }
+
+ /**
+ * Validates a big decimal property. The boundaries are inclusive.
+ */
+ public void validateBigDecimal(String key, boolean isOptional, BigDecimal min, BigDecimal max) {
+ validateComparable(key, isOptional, min, max, "decimal", BigDecimal::new);
+ }
+
+ /**
+ * Validates a byte property.
+ */
+ public void validateByte(String key, boolean isOptional) {
+ validateByte(key, isOptional, Byte.MIN_VALUE, Byte.MAX_VALUE);
+ }
+
+ /**
+ * Validates a byte property. The boundaries are inclusive.
+ */
+ public void validateByte(String key, boolean isOptional, byte min) {
+ validateByte(key, isOptional, min, Byte.MAX_VALUE);
+ }
+
+ /**
+ * Validates a byte property. The boundaries are inclusive.
+ */
+ public void validateByte(String key, boolean isOptional, byte min, byte max) {
+ validateComparable(key, isOptional, min, max, "byte", Byte::valueOf);
+ }
+
+ /**
+ * Validates a float property.
+ */
+ public void validateFloat(String key, boolean isOptional) {
+ validateFloat(key, isOptional, Float.MIN_VALUE, Float.MAX_VALUE);
+ }
+
+ /**
+ * Validates a float property. The boundaries are inclusive.
+ */
+ public void validateFloat(String key, boolean isOptional, float min) {
+ validateFloat(key, isOptional, min, Float.MAX_VALUE);
+ }
+
+ /**
+ * Validates a float property. The boundaries are inclusive.
+ */
+ public void validateFloat(String key, boolean isOptional, float min, float max) {
+ validateComparable(key, isOptional, min, max, "float", Float::valueOf);
+ }
+
+ /**
+ * Validates a short property.
+ */
+ public void validateShort(String key, boolean isOptional) {
+ validateShort(key, isOptional, Short.MIN_VALUE, Short.MAX_VALUE);
+ }
+
+ /**
+ * Validates a short property. The boundaries are inclusive.
+ */
+ public void validateShort(String key, boolean isOptional, short min) {
+ validateShort(key, isOptional, min, Short.MAX_VALUE);
+ }
+
+ /**
+ * Validates a short property. The boundaries are inclusive.
+ */
+ public void validateShort(String key, boolean isOptional, short min, short max) {
+ validateComparable(key, isOptional, min, max, "short", Short::valueOf);
+ }
+
+ /**
+ * Validation for fixed indexed properties.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ * </pre>
+ *
+ * <p>The subKeyValidation map must define e.g. "type" and "name" and a validation logic for the given full key.
+ */
+ public void validateFixedIndexedProperties(String key, boolean allowEmpty, Map<String, Consumer<String>> subKeyValidation) {
+ // determine max index
+ final int maxIndex = extractMaxIndex(key, "\\.(.*)");
+
+ if (maxIndex < 0 && !allowEmpty) {
+ throw new ValidationException("Property key '" + key + "' must not be empty.");
+ }
+
+ // validate
+ for (int i = 0; i < maxIndex; i++) {
+ for (Map.Entry<String, Consumer<String>> subKey : subKeyValidation.entrySet()) {
+ final String fullKey = key + '.' + i + '.' + subKey.getKey();
+ if (properties.containsKey(fullKey)) {
+ // run validation logic
+ subKey.getValue().accept(fullKey);
+ } else {
+ throw new ValidationException("Required property key '" + fullKey + "' is missing.");
+ }
+ }
+ }
+ }
+
+ /**
+ * Validates a table schema property.
+ */
+ public void validateTableSchema(String key, boolean isOptional) {
+ final Consumer<String> nameValidation = (name) -> validateString(name, false, 1);
+ final Consumer<String> typeValidation = (name) -> validateType(name, false, false);
+
+ final Map<String, Consumer<String>> subKeys = new HashMap<>();
+ subKeys.put(TABLE_SCHEMA_NAME, nameValidation);
+ subKeys.put(TABLE_SCHEMA_TYPE, typeValidation);
+
+ validateFixedIndexedProperties(
+ key,
+ isOptional,
+ subKeys
+ );
+ }
+
+ /**
+ * Validates a Flink {@link MemorySize}.
+ *
+ * <p>The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
+ */
+ public void validateMemorySize(String key, boolean isOptional, int precision) {
+ validateMemorySize(key, isOptional, precision, 0L, Long.MAX_VALUE);
+ }
+
+ /**
+ * Validates a Flink {@link MemorySize}. The boundaries are inclusive and in bytes.
+ *
+ * <p>The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
+ */
+ public void validateMemorySize(String key, boolean isOptional, int precision, long min) {
+ validateMemorySize(key, isOptional, precision, min, Long.MAX_VALUE);
+ }
+
+ /**
+ * Validates a Flink {@link MemorySize}. The boundaries are inclusive and in bytes.
+ *
+ * <p>The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
+ */
+ public void validateMemorySize(String key, boolean isOptional, int precision, long min, long max) {
+ Preconditions.checkArgument(precision > 0);
+
+ validateComparable(
+ key,
+ isOptional,
+ min,
+ max,
+ "memory size (in bytes)",
+ (value) -> {
+ final long bytes = MemorySize.parse(value, MemorySize.MemoryUnit.BYTES).getBytes();
+ if (bytes % precision != 0) {
+ throw new ValidationException(
+ "Memory size for key '" + key + "' must be a multiple of " + precision + " bytes but was: " + value);
+ }
+ return bytes;
+ }
+ );
+ }
+
+ /**
+ * Validates an enum property with a set of validation logic for each enum value.
+ */
+ public void validateEnum(String key, boolean isOptional, Map<String, Consumer<String>> enumValidation) {
+ validateOptional(
+ key,
+ isOptional,
+ (value) -> {
+ if (!enumValidation.containsKey(value)) {
+ throw new ValidationException(
+ "Unknown value for property '" + key + "'.\n" +
+ "Supported values are " + enumValidation.keySet() + " but was: " + value);
+ } else {
+ // run validation logic
+ enumValidation.get(value).accept(key);
+ }
+ });
+ }
+
+ /**
+ * Validates an enum property with a set of enum values.
+ */
+ public void validateEnumValues(String key, boolean isOptional, List<String> values) {
+ validateEnum(
+ key,
+ isOptional,
+ values.stream().collect(Collectors.toMap(v -> v, v -> noValidation())));
+ }
+
+ /**
+ * Validates a type property.
+ */
+ public void validateType(String key, boolean isOptional, boolean requireRow) {
+ validateOptional(
+ key,
+ isOptional,
+ (value) -> {
+ // we don't validate the string but let the parser do the work for us
+ // it throws a validation exception
+ final TypeInformation<?> typeInfo = TypeStringUtils.readTypeInfo(value);
+ if (requireRow && !(typeInfo instanceof RowTypeInfo)) {
+ throw new ValidationException(
+ "Row type information expected for key '" + key + "' but was: " + value);
+ }
+ });
+ }
+
+ /**
+ * Validates an array of values.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * primary-key.0 = field1
+ * primary-key.1 = field2
+ * </pre>
+ *
+ * <p>leads to: List(field1, field2)
+ *
+ * <p>or:
+ *
+ * <pre>
+ * primary-key = field1
+ * </pre>
+ *
+ * <p>The validation consumer gets the key of the current value e.g. "primary-key.1".
+ */
+ public void validateArray(String key, Consumer<String> elementValidation, int minLength) {
+ validateArray(key, elementValidation, minLength, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Validates an array of values.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ * primary-key.0 = field1
+ * primary-key.1 = field2
+ * </pre>
+ *
+ * <p>leads to: List(field1, field2)
+ *
+ * <p>or:
+ *
+ * <pre>
+ * primary-key = field1
+ * </pre>
+ *
+ * <p>The validation consumer gets the key of the current value e.g. "primary-key.1".
+ */
+ public void validateArray(String key, Consumer<String> elementValidation, int minLength, int maxLength) {
+
+ // determine max index
+ final int maxIndex = extractMaxIndex(key, "");
+
+ if (maxIndex < 0) {
+ // check for a single element array
+ if (properties.containsKey(key)) {
+ elementValidation.accept(key);
+ } else if (minLength > 0) {
+ throw new ValidationException("Could not find required property array for key '" + key + "'.");
+ }
+ } else {
+ // do not allow a single element array
+ if (properties.containsKey(key)) {
+ throw new ValidationException("Invalid property array for key '" + key + "'.");
+ }
+
+ final int size = maxIndex + 1;
+ if (size < minLength) {
+ throw new ValidationException(
+ "Array for key '" + key + "' must not have less than " + minLength + " elements but was: " + size);
+ }
+
+ if (size > maxLength) {
+ throw new ValidationException(
+ "Array for key '" + key + "' must not have more than " + maxLength + " elements but was: " + size);
+ }
+ }
+
+ // validate array elements
+ for (int i = 0; i < maxIndex; i++) {
+ final String fullKey = key + '.' + i;
+ if (properties.containsKey(fullKey)) {
+ // run validation logic
+ elementValidation.accept(fullKey);
+ } else {
+ throw new ValidationException(
+ "Required array element at index '" + i + "' for key '" + key + "' is missing.");
+ }
+ }
+ }
+
+ /**
+ * Validates that the given prefix is not included in these properties.
+ */
+ public void validatePrefixExclusion(String prefix) {
+ properties.keySet().stream()
+ .filter(k -> k.startsWith(prefix))
+ .findFirst()
+ .ifPresent((k) -> {
+ throw new ValidationException(
+ "Properties with prefix '" + prefix + "' are not allowed in this context. " +
+ "But property '" + k + "' was found.");
+ });
+ }
+
+ /**
+ * Validates that the given key is not included in these properties.
+ */
+ public void validateExclusion(String key) {
+ if (properties.containsKey(key)) {
+ throw new ValidationException("Property '" + key + "' is not allowed in this context.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Returns if the given key is contained.
+ */
+ public boolean containsKey(String key) {
+ return properties.containsKey(key);
+ }
+
+ /**
+ * Returns if a given prefix exists in the properties.
+ */
+ public boolean hasPrefix(String prefix) {
+ return properties.keySet().stream().anyMatch(k -> k.startsWith(prefix));
+ }
+
+ /**
+ * Returns the properties as a map copy.
+ */
+ public Map<String, String> asMap() {
+ final Map<String, String> copy = new HashMap<>(properties);
+ return Collections.unmodifiableMap(copy);
+ }
+
+ /**
+ * Returns the properties as a map copy with a prefix key.
+ */
+ public Map<String, String> asPrefixedMap(String prefix) {
+ return properties.entrySet().stream()
+ .collect(Collectors.toMap(e -> prefix + e.getKey(), Map.Entry::getValue));
+ }
+
+ /**
+ * Returns a new properties instance with the given keys removed.
+ */
+ public DescriptorProperties withoutKeys(List<String> keys) {
+ final Set<String> keySet = new HashSet<>(keys);
+ final DescriptorProperties copy = new DescriptorProperties(normalizeKeys);
+ properties.entrySet().stream()
+ .filter(e -> !keySet.contains(e.getKey()))
+ .forEach(e -> copy.properties.put(e.getKey(), e.getValue()));
+ return copy;
+ }
+
+ @Override
+ public String toString() {
+ return toString(properties);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DescriptorProperties that = (DescriptorProperties) o;
+ return Objects.equals(properties, that.properties);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(properties);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ private void put(String key, String value) {
+ if (properties.containsKey(key)) {
+ throw new ValidationException("Property already present: " + key);
+ }
+ if (normalizeKeys) {
+ properties.put(key.toLowerCase(), value);
+ } else {
+ properties.put(key, value);
+ }
+ }
+
+ private Optional<String> optionalGet(String key) {
+ return Optional.ofNullable(properties.get(key));
+ }
+
+ private void validateOptional(String key, boolean isOptional, Consumer<String> valueValidation) {
+ if (!properties.containsKey(key)) {
+ if (!isOptional) {
+ throw new ValidationException("Could not find required property '" + key + "'.");
+ }
+ } else {
+ final String value = properties.get(key);
+ valueValidation.accept(value);
+ }
+ }
+
+ private Supplier<TableException> exceptionSupplier(String key) {
+ return () -> {
+ throw new TableException(
+ "Property with key '" + key + "' could not be found. " +
+ "This is a bug because the validation logic should have checked that before.");
+ };
+ }
+
+ private int extractMaxIndex(String key, String suffixPattern) {
+ // extract index and property keys
+ final String escapedKey = Pattern.quote(key);
+ final Pattern pattern = Pattern.compile(escapedKey + "\\.(\\d+)" + suffixPattern);
+ final IntStream indexes = properties.keySet().stream()
+ .flatMapToInt(k -> {
+ final Matcher matcher = pattern.matcher(k);
+ if (matcher.find()) {
+ return IntStream.of(Integer.valueOf(matcher.group(1)));
+ }
+ return IntStream.empty();
+ });
+
+ // determine max index
+ return indexes.max().orElse(-1);
+ }
+
+ /**
+ * Validates a property by first parsing the string value to a comparable object. The boundaries are inclusive.
+ */
+ private <T extends Comparable<T>> void validateComparable(
+ String key,
+ boolean isOptional,
+ T min,
+ T max,
+ String typeName,
+ Function<String, T> parseFunction) {
+ if (!properties.containsKey(key)) {
+ if (!isOptional) {
+ throw new ValidationException("Could not find required property '" + key + "'.");
+ }
+ } else {
+ final String value = properties.get(key);
+ try {
+ final T parsed = parseFunction.apply(value);
+ if (parsed.compareTo(min) < 0 || parsed.compareTo(max) > 0) {
+ throw new ValidationException(
+ "Property '" + key + "' must be a " + typeName + " value between " + min + " and " + max + " but was: " + parsed);
+ }
+ } catch (Exception e) {
+ throw new ValidationException("Property '" + key + "' must be a " + typeName + " value but was: " + value);
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Returns an empty validation logic.
+ */
+ public static Consumer<String> noValidation() {
+ return EMPTY_CONSUMER;
+ }
+
+ public static String toString(String str) {
+ return EncodingUtils.escapeJava(str).replace("\\/", "/"); // '/' must not be escaped
+ }
+
+ public static String toString(String key, String value) {
+ return toString(key) + '=' + toString(value);
+ }
+
+ public static String toString(Map<String, String> propertyMap) {
+ return propertyMap.entrySet().stream()
+ .map(e -> toString(e.getKey(), e.getValue()))
+ .sorted()
+ .collect(Collectors.joining("\n"));
+ }
+}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
index 0ca17a5..7fc7de5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/exceptions.scala
@@ -21,6 +21,8 @@ package org.apache.flink.table.api
import org.apache.flink.table.descriptors.DescriptorProperties
import org.apache.flink.table.factories.TableFactory
+import _root_.scala.collection.JavaConverters._
+
/**
* Exception for all errors occurring during expression parsing.
*/
@@ -127,7 +129,7 @@ case class NoMatchingTableFactoryException(
|Reason: $message
|
|The following properties are requested:
- |${DescriptorProperties.toString(properties)}
+ |${DescriptorProperties.toString(properties.asJava)}
|
|The following factories have been considered:
|${factories.map(_.getClass.getName).mkString("\n")}
@@ -166,7 +168,7 @@ case class AmbiguousTableFactoryException(
|${matchingFactories.map(_.getClass.getName).mkString("\n")}
|
|The following properties are requested:
- |${DescriptorProperties.toString(properties)}
+ |${DescriptorProperties.toString(properties.asJava)}
|
|The following factories have been considered:
|${factories.map(_.getClass.getName).mkString("\n")}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
index 9576f34..cf5857e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
@@ -18,12 +18,13 @@
package org.apache.flink.table.catalog
-import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, readColumnStats}
import org.apache.flink.table.descriptors.StreamTableDescriptorValidator.{UPDATE_MODE, UPDATE_MODE_VALUE_APPEND, UPDATE_MODE_VALUE_RETRACT, UPDATE_MODE_VALUE_UPSERT}
import org.apache.flink.table.descriptors._
import org.apache.flink.table.factories.TableFactory
import org.apache.flink.table.plan.stats.TableStats
+import org.apache.flink.table.util.JavaScalaConversionUtil
+import org.apache.flink.table.util.JavaScalaConversionUtil.toScala
import scala.collection.JavaConverters._
@@ -294,7 +295,7 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = true,
isSink = false,
- DescriptorProperties.toJavaMap(this))
+ JavaScalaConversionUtil.toJavaMap(this))
}
/**
@@ -309,7 +310,7 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = false,
isSink = true,
- DescriptorProperties.toJavaMap(this))
+ JavaScalaConversionUtil.toJavaMap(this))
}
/**
@@ -324,7 +325,7 @@ class ExternalCatalogTableBuilder(private val connectorDescriptor: ConnectorDesc
isStreaming,
isSource = true,
isSink = true,
- DescriptorProperties.toJavaMap(this))
+ JavaScalaConversionUtil.toJavaMap(this))
}
// ----------------------------------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstanceValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstanceValidator.scala
index d9c3a02..43da1e1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstanceValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ClassInstanceValidator.scala
@@ -30,7 +30,7 @@ class ClassInstanceValidator(keyPrefix: String = EMPTY_PREFIX)
override def validateWithPrefix(keyPrefix: String, properties: DescriptorProperties): Unit = {
// check class name
- properties.validateString(s"$keyPrefix${ClassInstanceValidator.CLASS}", isOptional = false, 1)
+ properties.validateString(s"$keyPrefix${ClassInstanceValidator.CLASS}", false, 1)
// check constructor
val constructorPrefix = s"$keyPrefix${ClassInstanceValidator.CONSTRUCTOR}"
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
index 211d374..2b24c82 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
@@ -26,8 +26,8 @@ import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTO
class ConnectorDescriptorValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateString(CONNECTOR_TYPE, isOptional = false, minLen = 1)
- properties.validateInt(CONNECTOR_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateString(CONNECTOR_TYPE, false, 1)
+ properties.validateInt(CONNECTOR_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
index 7e69fea..e1204a1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
@@ -18,9 +18,12 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.{TableSchema, ValidationException}
import org.apache.flink.table.descriptors.CsvValidator._
+import org.apache.flink.table.utils.TypeStringUtils
import scala.collection.mutable
import scala.collection.JavaConverters._
@@ -33,7 +36,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
private var fieldDelim: Option[String] = None
private var lineDelim: Option[String] = None
private val schema: mutable.LinkedHashMap[String, String] =
- mutable.LinkedHashMap[String, String]()
+ mutable.LinkedHashMap[String, String]()
private var quoteCharacter: Option[Character] = None
private var commentPrefix: Option[String] = None
private var isIgnoreFirstLine: Option[Boolean] = None
@@ -69,8 +72,8 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
*/
def schema(schema: TableSchema): Csv = {
this.schema.clear()
- DescriptorProperties.normalizeTableSchema(schema).foreach {
- case (n, t) => field(n, t)
+ schema.getFieldNames.zip(schema.getFieldTypes).foreach { case (n, t) =>
+ field(n, t)
}
this
}
@@ -84,7 +87,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
* @param fieldType the type information of the field
*/
def field(fieldName: String, fieldType: TypeInformation[_]): Csv = {
- field(fieldName, DescriptorProperties.normalizeTypeInfo(fieldType))
+ field(fieldName, TypeStringUtils.writeTypeInfo(fieldType))
this
}
@@ -146,9 +149,17 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
override protected def addFormatProperties(properties: DescriptorProperties): Unit = {
fieldDelim.foreach(properties.putString(FORMAT_FIELD_DELIMITER, _))
lineDelim.foreach(properties.putString(FORMAT_LINE_DELIMITER, _))
- properties.putTableSchema(
+
+ val subKeys = util.Arrays.asList(
+ DescriptorProperties.TABLE_SCHEMA_NAME,
+ DescriptorProperties.TABLE_SCHEMA_TYPE)
+
+ val subValues = schema.map(e => util.Arrays.asList(e._1, e._2)).toList.asJava
+
+ properties.putIndexedFixedProperties(
FORMAT_FIELDS,
- schema.toIndexedSeq.map(DescriptorProperties.toJava[String, String]).asJava)
+ subKeys,
+ subValues)
quoteCharacter.foreach(properties.putCharacter(FORMAT_QUOTE_CHARACTER, _))
commentPrefix.foreach(properties.putString(FORMAT_COMMENT_PREFIX, _))
isIgnoreFirstLine.foreach(properties.putBoolean(FORMAT_IGNORE_FIRST_LINE, _))
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/CsvValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/CsvValidator.scala
index 3329b9f..9eeedb4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/CsvValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/CsvValidator.scala
@@ -28,14 +28,14 @@ class CsvValidator extends FormatDescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
super.validate(properties)
- properties.validateValue(FORMAT_TYPE, FORMAT_TYPE_VALUE, isOptional = false)
- properties.validateString(FORMAT_FIELD_DELIMITER, isOptional = true, minLen = 1)
- properties.validateString(FORMAT_LINE_DELIMITER, isOptional = true, minLen = 1)
- properties.validateString(FORMAT_QUOTE_CHARACTER, isOptional = true, minLen = 1, maxLen = 1)
- properties.validateString(FORMAT_COMMENT_PREFIX, isOptional = true, minLen = 1)
- properties.validateBoolean(FORMAT_IGNORE_FIRST_LINE, isOptional = true)
- properties.validateBoolean(FORMAT_IGNORE_PARSE_ERRORS, isOptional = true)
- properties.validateTableSchema(FORMAT_FIELDS, isOptional = false)
+ properties.validateValue(FORMAT_TYPE, FORMAT_TYPE_VALUE, false)
+ properties.validateString(FORMAT_FIELD_DELIMITER, true, 1)
+ properties.validateString(FORMAT_LINE_DELIMITER, true, 1)
+ properties.validateString(FORMAT_QUOTE_CHARACTER, true, 1, 1)
+ properties.validateString(FORMAT_COMMENT_PREFIX, true, 1)
+ properties.validateBoolean(FORMAT_IGNORE_FIRST_LINE, true)
+ properties.validateBoolean(FORMAT_IGNORE_PARSE_ERRORS, true)
+ properties.validateTableSchema(FORMAT_FIELDS, false)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
deleted file mode 100644
index 6288ca5..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
+++ /dev/null
@@ -1,1601 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import java.io.Serializable
-import java.lang.{Boolean => JBoolean, Byte => JByte, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong, Short => JShort}
-import java.math.{BigDecimal => JBigDecimal}
-import java.util
-import java.util.function.{BiConsumer, Consumer, Function, Supplier}
-import java.util.regex.Pattern
-import java.util.{Optional, List => JList, Map => JMap}
-
-import org.apache.commons.codec.binary.Base64
-import org.apache.commons.lang.StringEscapeUtils
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
-import org.apache.flink.api.java.typeutils.RowTypeInfo
-import org.apache.flink.configuration.MemorySize
-import org.apache.flink.table.api.{TableException, TableSchema, ValidationException}
-import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema, toJava}
-import org.apache.flink.table.utils.TypeStringUtils
-import org.apache.flink.util.Preconditions.checkNotNull
-import org.apache.flink.util.{InstantiationUtil, Preconditions}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-/**
- * Utility class for having a unified string-based representation of Table API related classes
- * such as [[TableSchema]], [[TypeInformation]], etc.
- *
- * '''Note to implementers''': Please try to reuse key names as much as possible. Key-names
- * should be hierarchical and lower case. Use "-" instead of dots or camel case.
- * E.g., connector.schema.start-from = from-earliest. Try not to use the higher level in a
- * key-name. E.g., instead of connector.kafka.kafka-version use connector.kafka.version.
- *
- * Existing property keys can not be overwritten. This ensures a property is defined exactly once.
- *
- * @param normalizeKeys flag that indicates if keys should be normalized (this flag is
- * necessary for backwards compatibility)
- */
-class DescriptorProperties(normalizeKeys: Boolean = true) {
-
- private val properties: mutable.Map[String, String] = new mutable.HashMap[String, String]()
-
- /**
- * Adds a set of properties.
- */
- def putProperties(properties: JMap[String, String]): Unit = {
- properties.asScala.foreach { case (k, v) =>
- put(k, v)
- }
- }
-
- /**
- * Adds a set of descriptor properties.
- */
- def putProperties(properties: DescriptorProperties): Unit = {
- properties.properties.foreach { case (k, v) =>
- put(k, v)
- }
- }
-
- /**
- * Adds a class under the given key.
- */
- def putClass(key: String, clazz: Class[_]): Unit = {
- checkNotNull(key)
- checkNotNull(clazz)
- val error = InstantiationUtil.checkForInstantiationError(clazz)
- if (error != null) {
- throw new ValidationException(s"Class '${clazz.getName}' is not supported: $error")
- }
- put(key, clazz.getName)
- }
-
- /**
- * Adds a string under the given key.
- */
- def putString(key: String, str: String): Unit = {
- checkNotNull(key)
- checkNotNull(str)
- put(key, str)
- }
-
- /**
- * Adds a boolean under the given key.
- */
- def putBoolean(key: String, b: Boolean): Unit = {
- checkNotNull(key)
- put(key, b.toString)
- }
-
- /**
- * Adds a long under the given key.
- */
- def putLong(key: String, l: Long): Unit = {
- checkNotNull(key)
- put(key, l.toString)
- }
-
- /**
- * Adds an integer under the given key.
- */
- def putInt(key: String, i: Int): Unit = {
- checkNotNull(key)
- put(key, i.toString)
- }
-
- /**
- * Adds a character under the given key.
- */
- def putCharacter(key: String, c: Character): Unit = {
- checkNotNull(key)
- checkNotNull(c)
- put(key, c.toString)
- }
-
- /**
- * Adds a table schema under the given key.
- */
- def putTableSchema(key: String, schema: TableSchema): Unit = {
- checkNotNull(key)
- checkNotNull(schema)
- putTableSchema(key, normalizeTableSchema(schema))
- }
-
- /**
- * Adds a Flink [[MemorySize]] under the given key.
- */
- def putMemorySize(key: String, size: MemorySize): Unit = {
- checkNotNull(key)
- checkNotNull(size)
- put(key, size.toString)
- }
-
- /**
- * Adds a table schema under the given key.
- */
- def putTableSchema(key: String, nameAndType: JList[JTuple2[String, String]]): Unit = {
- putTableSchema(key, nameAndType.asScala.map(t => (t.f0, t.f1)))
- }
-
- /**
- * Adds an indexed sequence of properties (with sub-properties) under a common key.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG, schema.fields.1.name = test2
- *
- * The arity of each propertyValue must match the arity of propertyKeys.
- */
- def putIndexedFixedProperties(
- key: String,
- propertyKeys: JList[String],
- propertyValues: JList[JList[String]])
- : Unit = {
- checkNotNull(key)
- checkNotNull(propertyValues)
- putIndexedFixedProperties(key, propertyKeys.asScala, propertyValues.asScala.map(_.asScala))
- }
-
- /**
- * Adds an indexed mapping of properties under a common key.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.name = test2
- *
- * The arity of the propertySets can differ.
- */
- def putIndexedVariableProperties(
- key: String,
- propertySets: JList[JMap[String, String]])
- : Unit = {
- checkNotNull(key)
- checkNotNull(propertySets)
- putIndexedVariableProperties(key, propertySets.asScala.map(_.asScala.toMap))
- }
-
- // ----------------------------------------------------------------------------------------------
-
- /**
- * Returns a string value under the given key if it exists.
- */
- def getOptionalString(key: String): Optional[String] = toJava(properties.get(key))
-
- /**
- * Returns a string value under the given existing key.
- */
- def getString(key: String): String = {
- get(key)
- }
-
- /**
- * Returns a character value under the given key if it exists.
- */
- def getOptionalCharacter(key: String): Optional[Character] = {
- val value = properties.get(key).map { c =>
- if (c.length != 1) {
- throw new ValidationException(s"The value of $key must only contain one character.")
- }
- Char.box(c.charAt(0))
- }
- toJava(value)
- }
-
- /**
- * Returns a character value under the given existing key.
- */
- def getCharacter(key: String): Char = {
- getOptionalCharacter(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a class value under the given key if it exists.
- */
- def getOptionalClass[T](key: String, superClass: Class[T]): Optional[Class[T]] = {
- val value = properties.get(key).map { name =>
- val clazz = try {
- Class.forName(
- name,
- true,
- Thread.currentThread().getContextClassLoader).asInstanceOf[Class[T]]
- } catch {
- case e: Exception =>
- throw new ValidationException(s"Could not get class '$name' for key '$key'.", e)
- }
- if (!superClass.isAssignableFrom(clazz)) {
- throw new ValidationException(s"Class '$name' does not extend from the required " +
- s"class '${superClass.getName}' for key '$key'.")
- }
- clazz
- }
- toJava(value)
- }
-
- /**
- * Returns a class value under the given existing key.
- */
- def getClass[T](key: String, superClass: Class[T]): Class[T] = {
- getOptionalClass(key, superClass).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a big decimal value under the given key if it exists.
- */
- def getOptionalBigDecimal(key: String): Optional[JBigDecimal] = {
- val value = properties.get(key).map(new JBigDecimal(_))
- toJava(value)
- }
-
- /**
- * Returns a big decimal value under the given existing key.
- */
- def getBigDecimal(key: String): JBigDecimal = {
- getOptionalBigDecimal(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a boolean value under the given key if it exists.
- */
- def getOptionalBoolean(key: String): Optional[JBoolean] = {
- val value = properties.get(key).map(JBoolean.parseBoolean).map(Boolean.box)
- toJava(value)
- }
-
- /**
- * Returns a boolean value under the given existing key.
- */
- def getBoolean(key: String): Boolean = {
- getOptionalBoolean(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a byte value under the given key if it exists.
- */
- def getOptionalByte(key: String): Optional[JByte] = {
- val value = properties.get(key).map(JByte.parseByte).map(Byte.box)
- toJava(value)
- }
-
- /**
- * Returns a byte value under the given existing key.
- */
- def getByte(key: String): Byte = {
- getOptionalByte(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a double value under the given key if it exists.
- */
- def getOptionalDouble(key: String): Optional[JDouble] = {
- val value = properties.get(key).map(JDouble.parseDouble).map(Double.box)
- toJava(value)
- }
-
- /**
- * Returns a double value under the given existing key.
- */
- def getDouble(key: String): Double = {
- getOptionalDouble(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a float value under the given key if it exists.
- */
- def getOptionalFloat(key: String): Optional[JFloat] = {
- val value = properties.get(key).map(JFloat.parseFloat).map(Float.box)
- toJava(value)
- }
-
- /**
- * Returns a float value under the given given existing key.
- */
- def getFloat(key: String): Float = {
- getOptionalFloat(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns an integer value under the given key if it exists.
- */
- def getOptionalInt(key: String): Optional[JInt] = {
- val value = properties.get(key).map(JInt.parseInt).map(Int.box)
- toJava(value)
- }
-
- /**
- * Returns an integer value under the given existing key.
- */
- def getInt(key: String): Int = {
- getOptionalInt(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a long value under the given key if it exists.
- */
- def getOptionalLong(key: String): Optional[JLong] = {
- val value = properties.get(key).map(JLong.parseLong).map(Long.box)
- toJava(value)
- }
-
- /**
- * Returns a long value under the given existing key.
- */
- def getLong(key: String): Long = {
- getOptionalLong(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a short value under the given key if it exists.
- */
- def getOptionalShort(key: String): Optional[JShort] = {
- val value = properties.get(key).map(JShort.parseShort).map(Short.box)
- toJava(value)
- }
-
- /**
- * Returns a short value under the given existing key.
- */
- def getShort(key: String): Short = {
- getOptionalShort(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns the type information under the given key if it exists.
- */
- def getOptionalType(key: String): Optional[TypeInformation[_]] = {
- val value = properties.get(key).map(TypeStringUtils.readTypeInfo)
- toJava(value)
- }
-
- /**
- * Returns the type information under the given existing key.
- */
- def getType(key: String): TypeInformation[_] = {
- getOptionalType(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a table schema under the given key if it exists.
- */
- def getOptionalTableSchema(key: String): Optional[TableSchema] = {
- // filter for number of columns
- val fieldCount = properties
- .filterKeys(k => k.startsWith(key) && k.endsWith(s".$NAME"))
- .size
-
- if (fieldCount == 0) {
- return toJava(None)
- }
- // validate fields and build schema
- val schemaBuilder = TableSchema.builder()
- for (i <- 0 until fieldCount) {
- val name = s"$key.$i.$NAME"
- val tpe = s"$key.$i.$TYPE"
- schemaBuilder.field(
- properties.getOrElse(name, throw new ValidationException(s"Invalid table schema. " +
- s"Could not find name for field '$key.$i'.")
- ),
- TypeStringUtils.readTypeInfo(
- properties.getOrElse(tpe, throw new ValidationException(s"Invalid table schema. " +
- s"Could not find type for field '$key.$i'."))
- )
- )
- }
- toJava(Some(schemaBuilder.build()))
- }
-
- /**
- * Returns a table schema under the given existing key.
- */
- def getTableSchema(key: String): TableSchema = {
- getOptionalTableSchema(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a Flink [[MemorySize]] under the given key if it exists.
- */
- def getOptionalMemorySize(key: String): Optional[MemorySize] = {
- val value = properties.get(key).map(MemorySize.parse(_, MemorySize.MemoryUnit.BYTES))
- toJava(value)
- }
-
- /**
- * Returns a Flink [[MemorySize]] under the given existing key.
- */
- def getMemorySize(key: String): MemorySize = {
- getOptionalMemorySize(key).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns the property keys of fixed indexed properties.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG, schema.fields.1.name = test2
- *
- * getFixedIndexedProperties("schema.fields", List("type", "name")) leads to:
- *
- * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
- * 1: Map("type" -> "schema.fields.1.type", "name" -> "schema.fields.1.name")
- */
- def getFixedIndexedProperties(
- key: String,
- propertyKeys: JList[String])
- : JList[JMap[String, String]] = {
-
- val keys = propertyKeys.asScala
-
- // determine max index
- val maxIndex = extractMaxIndex(key, "\\.(.*)")
-
- // validate and create result
- val list = new util.ArrayList[JMap[String, String]]()
- for (i <- 0 to maxIndex) {
- val map = new util.HashMap[String, String]()
-
- keys.foreach { subKey =>
- val fullKey = s"$key.$i.$subKey"
- // check for existence of full key
- if (!containsKey(fullKey)) {
- throw exceptionSupplier(fullKey).get()
- }
- map.put(subKey, fullKey)
- }
-
- list.add(map)
- }
- list
- }
-
- /**
- * Returns the property keys of variable indexed properties.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG
- *
- * getFixedIndexedProperties("schema.fields", List("type")) leads to:
- *
- * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
- * 1: Map("type" -> "schema.fields.1.type")
- */
- def getVariableIndexedProperties(
- key: String,
- requiredKeys: JList[String])
- : JList[JMap[String, String]] = {
-
- val keys = requiredKeys.asScala
-
- // filter for index
- val escapedKey = Pattern.quote(key)
- val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)(\\.)?(.*)")
-
- // extract index and property keys
- val indexes = properties.keys.flatMap { k =>
- val matcher = pattern.matcher(k)
- if (matcher.find()) {
- Some((JInt.parseInt(matcher.group(1)), matcher.group(3)))
- } else {
- None
- }
- }
-
- // determine max index
- val maxIndex = indexes.map(_._1).reduceOption(_ max _).getOrElse(-1)
-
- // validate and create result
- val list = new util.ArrayList[JMap[String, String]]()
- for (i <- 0 to maxIndex) {
- val map = new util.HashMap[String, String]()
-
- // check and add required keys
- keys.foreach { subKey =>
- val fullKey = s"$key.$i.$subKey"
- // check for existence of full key
- if (!containsKey(fullKey)) {
- throw exceptionSupplier(fullKey).get()
- }
- map.put(subKey, fullKey)
- }
-
- // add optional keys
- indexes.filter(_._1 == i).foreach { case (_, subKey) =>
- val fullKey = s"$key.$i.$subKey"
- map.put(subKey, fullKey)
- }
-
- list.add(map)
- }
- list
- }
-
- /**
- * Returns all properties under a given key that contains an index in between.
- *
- * E.g. rowtime.0.name -> returns all rowtime.#.name properties
- */
- def getIndexedProperty(key: String, property: String): JMap[String, String] = {
- val escapedKey = Pattern.quote(key)
- properties.filterKeys(k => k.matches(s"$escapedKey\\.\\d+\\.$property")).asJava
- }
-
- /**
- * Returns all array elements under a given key if it exists.
- *
- * For example:
- *
- * primary-key.0 = field1
- * primary-key.1 = field2
- *
- * leads to: List(field1, field2)
- *
- * or:
- *
- * primary-key = field1
- *
- * leads to: List(field1)
- *
- * The key mapper gets the key of the current value e.g. "primary-key.1".
- */
- def getOptionalArray[E](key: String, keyMapper: Function[String, E]): Optional[JList[E]] = {
- // determine max index
- val maxIndex = extractMaxIndex(key)
-
- if (maxIndex < 0) {
- // check for a single element array
- if (containsKey(key)) {
- val value = keyMapper.apply(key)
- Optional.of(util.Collections.singletonList(value))
- } else {
- Optional.empty()
- }
- } else {
- val list = new util.ArrayList[E]()
- for (i <- 0 to maxIndex) {
- val fullKey = s"$key.$i"
- if (containsKey(fullKey)) {
- list.add(keyMapper.apply(fullKey))
- } else {
- throw exceptionSupplier(fullKey).get()
- }
- }
- Optional.of(list)
- }
- }
-
- /**
- * Returns all array elements under a given existing key.
- */
- def getArray[E](key: String, keyMapper: Function[String, E]): JList[E] = {
- getOptionalArray(key, keyMapper).orElseThrow(exceptionSupplier(key))
- }
-
- /**
- * Returns a prefix subset of properties.
- */
- def getPrefix(prefixKey: String): JMap[String, String] = {
- val prefix = prefixKey + '.'
- properties.filterKeys(_.startsWith(prefix)).toSeq.map{ case (k, v) =>
- k.substring(prefix.length) -> v // remove prefix
- }.toMap.asJava
- }
-
- /**
- * Returns if a key is exactly equal to the given value.
- */
- def isValue(key: String, value: String): Boolean = {
- getString(key) == value
- }
-
- // ----------------------------------------------------------------------------------------------
-
- /**
- * Validates a string property.
- */
- def validateString(
- key: String,
- isOptional: Boolean)
- : Unit = {
- validateString(key, isOptional, 0, Integer.MAX_VALUE)
- }
-
- /**
- * Validates a string property. The boundaries are inclusive.
- */
- def validateString(
- key: String,
- isOptional: Boolean,
- minLen: Int) // inclusive
- : Unit = {
- validateString(key, isOptional, minLen, Integer.MAX_VALUE)
- }
-
- /**
- * Validates a string property. The boundaries are inclusive.
- */
- def validateString(
- key: String,
- isOptional: Boolean,
- minLen: Int, // inclusive
- maxLen: Int) // inclusive
- : Unit = {
-
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- val len = properties(key).length
- if (len < minLen || len > maxLen) {
- throw new ValidationException(
- s"Property '$key' must have a length between $minLen and $maxLen but " +
- s"was: ${properties(key)}")
- }
- }
- }
-
- /**
- * Validates an integer property.
- */
- def validateInt(
- key: String,
- isOptional: Boolean)
- : Unit = {
- validateInt(key, isOptional, Int.MinValue, Int.MaxValue)
- }
-
- /**
- * Validates an integer property. The boundaries are inclusive.
- */
- def validateInt(
- key: String,
- isOptional: Boolean,
- min: Int) // inclusive
- : Unit = {
- validateInt(key, isOptional, min, Int.MaxValue)
- }
-
- /**
- * Validates an integer property. The boundaries are inclusive.
- */
- def validateInt(
- key: String,
- isOptional: Boolean,
- min: Int, // inclusive
- max: Int) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new Integer(min), new Integer(max), Integer.valueOf)
- }
-
- /**
- * Validates a long property.
- */
- def validateLong(
- key: String,
- isOptional: Boolean)
- : Unit = {
- validateLong(key, isOptional, Long.MinValue, Long.MaxValue)
- }
-
- /**
- * Validates a long property. The boundaries are inclusive.
- */
- def validateLong(
- key: String,
- isOptional: Boolean,
- min: Long) // inclusive
- : Unit = {
- validateLong(key, isOptional, min, Long.MaxValue)
- }
-
- /**
- * Validates a long property. The boundaries are inclusive.
- */
- def validateLong(
- key: String,
- isOptional: Boolean,
- min: Long, // inclusive
- max: Long) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new JLong(min), new JLong(max), JLong.valueOf)
- }
-
- /**
- * Validates that a certain value is present under the given key.
- */
- def validateValue(key: String, value: String, isOptional: Boolean): Unit = {
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- if (properties(key) != value) {
- throw new ValidationException(
- s"Could not find required value '$value' for property '$key'.")
- }
- }
- }
-
- /**
- * Validates that a boolean value is present under the given key.
- */
- def validateBoolean(key: String, isOptional: Boolean): Unit = {
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- val value = properties(key)
- if (!value.equalsIgnoreCase("true") && !value.equalsIgnoreCase("false")) {
- throw new ValidationException(
- s"Property '$key' must be a boolean value (true/false) but was: $value")
- }
- }
- }
-
- /**
- * Validates a double property.
- */
- def validateDouble(
- key: String,
- isOptional: Boolean)
- : Unit = {
- validateDouble(key, isOptional, Double.MinValue, Double.MaxValue)
- }
-
- /**
- * Validates a double property. The boundaries are inclusive.
- */
- def validateDouble(
- key: String,
- isOptional: Boolean,
- min: Double) // inclusive
- : Unit = {
- validateDouble(key, isOptional, min, Double.MaxValue)
- }
-
- /**
- * Validates a double property. The boundaries are inclusive.
- */
- def validateDouble(
- key: String,
- isOptional: Boolean,
- min: Double, // inclusive
- max: Double) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new JDouble(min), new JDouble(max), JDouble.valueOf)
- }
-
- /**
- * Validates a big decimal property.
- */
- def validateBigDecimal(
- key: String,
- isOptional: Boolean)
- : Unit = {
-
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- try {
- new JBigDecimal(properties(key))
- } catch {
- case _: NumberFormatException =>
- throw new ValidationException(
- s"Property '$key' must be a big decimal value but was: ${properties(key)}")
- }
- }
- }
-
- /**
- * Validates a big decimal property. The boundaries are inclusive.
- */
- def validateBigDecimal(
- key: String,
- isOptional: Boolean,
- min: JBigDecimal, // inclusive
- max: JBigDecimal) // inclusive
- : Unit = {
- validateComparable(
- key,
- isOptional,
- min,
- max,
- (value: String) => new JBigDecimal(value))
- }
-
- /**
- * Validates a byte property.
- */
- def validateByte(
- key: String,
- isOptional: Boolean): Unit = validateByte(key, isOptional, Byte.MinValue, Byte.MaxValue)
-
- /**
- * Validates a byte property. The boundaries are inclusive.
- */
- def validateByte(
- key: String,
- isOptional: Boolean,
- min: Byte) // inclusive
- : Unit = {
- validateByte(key, isOptional, min, Byte.MaxValue)
- }
-
- /**
- * Validates a byte property. The boundaries are inclusive.
- */
- def validateByte(
- key: String,
- isOptional: Boolean,
- min: Byte, // inclusive
- max: Byte) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new JByte(min), new JByte(max), JByte.valueOf)
- }
-
- /**
- * Validates a float property.
- */
- def validateFloat(
- key: String,
- isOptional: Boolean): Unit = validateFloat(key, isOptional, Float.MinValue, Float.MaxValue)
-
- /**
- * Validates a float property. The boundaries are inclusive.
- */
- def validateFloat(
- key: String,
- isOptional: Boolean,
- min: Float) // inclusive
- : Unit = {
- validateFloat(key, isOptional, min, Float.MaxValue)
- }
-
- /**
- * Validates a float property. The boundaries are inclusive.
- */
- def validateFloat(
- key: String,
- isOptional: Boolean,
- min: Float, // inclusive
- max: Float) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new JFloat(min), new JFloat(max), JFloat.valueOf)
- }
-
- /**
- * Validates a short property.
- */
- def validateShort(
- key: String,
- isOptional: Boolean): Unit = validateFloat(key, isOptional, Short.MinValue, Short.MaxValue)
-
- /**
- * Validates a short property. The boundaries are inclusive.
- */
- def validateShort(
- key: String,
- isOptional: Boolean,
- min: Short) // inclusive
- : Unit = {
- validateShort(key, isOptional, min, Short.MaxValue)
- }
-
- /**
- * Validates a short property. The boundaries are inclusive.
- */
- def validateShort(
- key: String,
- isOptional: Boolean,
- min: Short, // inclusive
- max: Short) // inclusive
- : Unit = {
- validateComparable(key, isOptional, new JShort(min), new JShort(max), JShort.valueOf)
- }
-
- /**
- * Validation for variable indexed properties.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG
- *
- * The propertyKeys map defines e.g. "type" and a validation logic for the given full key.
- *
- * The validation consumer takes the current prefix e.g. "schema.fields.1.".
- */
- def validateVariableIndexedProperties(
- key: String,
- allowEmpty: Boolean,
- propertyKeys: JMap[String, Consumer[String]],
- requiredKeys: JList[String])
- : Unit = {
-
- val keys = propertyKeys.asScala
-
- // determine max index
- val maxIndex = extractMaxIndex(key, "\\.(.*)")
-
- if (maxIndex < 0 && !allowEmpty) {
- throw new ValidationException(s"Property key '$key' must not be empty.")
- }
-
- // validate
- for (i <- 0 to maxIndex) {
- keys.foreach { case (subKey, validation) =>
- val fullKey = s"$key.$i.$subKey"
- // only validate if it exists
- if (properties.contains(fullKey)) {
- validation.accept(s"$key.$i.")
- } else {
- // check if it is required
- if (requiredKeys.contains(subKey)) {
- throw new ValidationException(s"Required property key '$fullKey' is missing.")
- }
- }
- }
- }
- }
-
- /**
- * Validation for fixed indexed properties.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG, schema.fields.1.name = test2
- *
- * The propertyKeys map must define e.g. "type" and "name" and a validation logic for the
- * given full key.
- */
- def validateFixedIndexedProperties(
- key: String,
- allowEmpty: Boolean,
- propertyKeys: JMap[String, Consumer[String]])
- : Unit = {
-
- validateVariableIndexedProperties(
- key,
- allowEmpty,
- propertyKeys,
- new util.ArrayList(propertyKeys.keySet()))
- }
-
- /**
- * Validates a table schema property.
- */
- def validateTableSchema(key: String, isOptional: Boolean): Unit = {
- val nameValidation = (prefix: String) => {
- validateString(prefix + NAME, isOptional = false, minLen = 1)
- }
- val typeValidation = (prefix: String) => {
- validateType(prefix + TYPE, requireRow = false, isOptional = false)
- }
-
- validateFixedIndexedProperties(
- key,
- isOptional,
- Map(
- NAME -> toJava(nameValidation),
- TYPE -> toJava(typeValidation)
- ).asJava
- )
- }
-
- /**
- * Validates a Flink [[MemorySize]].
- *
- * The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
- */
- def validateMemorySize(
- key: String,
- isOptional: Boolean,
- precision: Int)
- : Unit = {
- validateMemorySize(key, isOptional, precision, 0, Long.MaxValue)
- }
-
- /**
- * Validates a Flink [[MemorySize]]. The boundaries are inclusive and in bytes.
- *
- * The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
- */
- def validateMemorySize(
- key: String,
- isOptional: Boolean,
- precision: Int,
- min: Long) // inclusive
- : Unit = {
- validateMemorySize(key, isOptional, precision, min, Long.MaxValue)
- }
-
- /**
- * Validates a Flink [[MemorySize]]. The boundaries are inclusive and in bytes.
- *
- * The precision defines the allowed minimum unit in bytes (e.g. 1024 would only allow KB).
- */
- def validateMemorySize(
- key: String,
- isOptional: Boolean,
- precision: Int,
- min: Long, // inclusive
- max: Long) // inclusive
- : Unit = {
- Preconditions.checkArgument(precision > 0)
-
- validateComparable(
- key,
- isOptional,
- Long.box(min),
- Long.box(max),
- "memory size (in bytes)",
- (value: String) => {
- val bytes = Long.box(MemorySize.parse(value, MemorySize.MemoryUnit.BYTES).getBytes)
- if (bytes % precision != 0) {
- throw new ValidationException(
- s"Memory size for key '$key' must be a multiple of $precision bytes but was: $value")
- }
- bytes
- })
- }
-
- /**
- * Validates a enum property with a set of validation logic for each enum value.
- */
- def validateEnum(
- key: String,
- isOptional: Boolean,
- enumToValidation: JMap[String, Consumer[String]])
- : Unit = {
-
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- val value = properties(key)
- if (!enumToValidation.containsKey(value)) {
- throw new ValidationException(s"Unknown value for property '$key'. " +
- s"Supported values [${enumToValidation.keySet().asScala.mkString(", ")}] but was: $value")
- } else {
- // run validation logic
- enumToValidation.get(value).accept(key)
- }
- }
- }
-
- /**
- * Validates a enum property with a set of enum values.
- */
- def validateEnumValues(key: String, isOptional: Boolean, values: JList[String]): Unit = {
- validateEnum(key, isOptional, values.asScala.map((_, noValidation())).toMap.asJava)
- }
-
- /**
- * Validates a type property.
- */
- def validateType(key: String, requireRow: Boolean, isOptional: Boolean): Unit = {
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- // we don't validate the string but let the parser do the work for us
- // it throws a validation exception
- val info = TypeStringUtils.readTypeInfo(properties(key))
- if (requireRow && !info.isInstanceOf[RowTypeInfo]) {
- throw new ValidationException(
- s"Row type information expected for '$key' but was: ${properties(key)}")
- }
- }
- }
-
- /**
- * Validates an array of values.
- *
- * For example:
- *
- * primary-key.0 = field1
- * primary-key.1 = field2
- *
- * or:
- *
- * primary-key = field1
- *
- * The validation consumer gets the key of the current value e.g. "primary-key.1".
- */
- def validateArray(
- key: String,
- validationConsumer: Consumer[String],
- minLength: Int) // inclusive
- : Unit = {
- validateArray(key, validationConsumer, minLength, Integer.MAX_VALUE)
- }
-
- /**
- * Validates an array of values.
- *
- * For example:
- *
- * primary-key.0 = field1
- * primary-key.1 = field2
- *
- * or:
- *
- * primary-key = field1
- *
- * The validation consumer gets the key of the current value e.g. "primary-key.1".
- */
- def validateArray(
- key: String,
- validationConsumer: Consumer[String],
- minLength: Int, // inclusive
- maxLength: Int) // inclusive
- : Unit = {
-
- // determine max index
- val maxIndex = extractMaxIndex(key)
-
- if (maxIndex < 0) {
- // check for a single element array
- if (containsKey(key)) {
- validationConsumer.accept(key)
- } else if (minLength > 0) {
- throw new ValidationException(s"Could not find required property array for key '$key'.")
- }
- } else {
- // do not allow a single element array
- if (containsKey(key)) {
- throw new ValidationException(s"Invalid property array for key '$key'.")
- }
-
- val size = maxIndex + 1
- if (size < minLength) {
- throw new ValidationException(
- s"Array for key '$key' must not have less than $minLength elements but was: $size")
- }
-
- if (size > maxLength) {
- throw new ValidationException(
- s"Array for key '$key' must not have more than $maxLength elements but was: $size")
- }
-
- // validate array elements
- for (i <- 0 to maxIndex) {
- val fullKey = s"$key.$i"
- // only validate if it exists
- if (containsKey(fullKey)) {
- validationConsumer.accept(fullKey)
- } else {
- throw new ValidationException(
- s"Required array element at index '$i' for key '$key' is missing.")
- }
- }
- }
- }
-
- /**
- * Validates that the given prefix is not included in these properties.
- */
- def validatePrefixExclusion(prefix: String): Unit = {
- val invalidField = properties.find(_._1.startsWith(prefix))
- if (invalidField.isDefined) {
- throw new ValidationException(
- s"Property '${invalidField.get._1}' is not allowed in this context.")
- }
- }
-
- /**
- * Validates that the given key is not included in these properties.
- */
- def validateExclusion(key: String): Unit = {
- if (properties.contains(key)) {
- throw new ValidationException(s"Property '$key' is not allowed in this context.")
- }
- }
-
- // ----------------------------------------------------------------------------------------------
-
- /**
- * Returns if any property contains parts of a given string.
- */
- def containsString(str: String): Boolean = {
- properties.exists(e => e._1.contains(str))
- }
-
- /**
- * Returns if the given key is contained.
- */
- def containsKey(key: String): Boolean = {
- properties.contains(key)
- }
-
- /**
- * Returns if a given prefix exists in the properties.
- */
- def hasPrefix(prefix: String): Boolean = {
- properties.exists(e => e._1.startsWith(prefix))
- }
-
- /**
- * Returns the properties as a Map copy.
- */
- def asMap: JMap[String, String] = {
- properties.toMap.asJava
- }
-
- /**
- * Returns the properties as a Map copy with a prefix key.
- */
- def asPrefixedMap(prefixKey: String): JMap[String, String] = {
- properties.map { case (key, value) =>
- (prefixKey + key, value)
- }.toMap.asJava
- }
-
- /**
- * Returns a new properties instance with the given keys removed.
- */
- def withoutKeys(keys: JList[String]): DescriptorProperties = {
- val copyProperties = new DescriptorProperties(true)
- properties
- .filterKeys(!keys.contains(_))
- .foreach { case (k, v) =>
- copyProperties.properties.put(k, v)
- }
- copyProperties
- }
-
- override def toString: String = {
- DescriptorProperties.toString(properties.toMap)
- }
-
- override def equals(other: Any): Boolean = other match {
- case that: DescriptorProperties => properties == that.properties
- case _ => false
- }
-
- override def hashCode(): Int = properties.hashCode()
-
- // ----------------------------------------------------------------------------------------------
-
- /**
- * Returns an empty validation logic.
- */
- def noValidation(): Consumer[String] = DescriptorProperties.emptyConsumer
-
- def exceptionSupplier(key: String): Supplier[TableException] = new Supplier[TableException] {
- override def get(): TableException = {
- new TableException(s"Property with key '$key' could not be found. " +
- s"This is a bug because the validation logic should have checked that before.")
- }
- }
-
- // ----------------------------------------------------------------------------------------------
-
- /**
- * Adds a property.
- */
- private def put(key: String, value: String): Unit = {
- if (properties.contains(key)) {
- throw new ValidationException(s"Property already present: $key")
- }
- if (normalizeKeys) {
- properties.put(key.toLowerCase, value)
- } else {
- properties.put(key, value)
- }
- }
-
- /**
- * Gets an existing property.
- */
- private def get(key: String): String = {
- properties.getOrElse(
- key,
- throw exceptionSupplier(key).get())
- }
-
- /**
- * Raw access to the underlying properties map for testing purposes.
- */
- private[flink] def unsafePut(key: String, value: String): Unit = {
- properties.put(key, value)
- }
-
- /**
- * Raw access to the underlying properties map for testing purposes.
- */
- private[flink] def unsafeRemove(key: String): Unit = {
- properties.remove(key)
- }
-
- /**
- * Adds a table schema under the given key.
- */
- private def putTableSchema(key: String, nameAndType: Seq[(String, String)]): Unit = {
- putIndexedFixedProperties(
- key,
- Seq(NAME, TYPE),
- nameAndType.map(t => Seq(t._1, t._2))
- )
- }
-
- /**
- * Adds an indexed sequence of properties (with sub-properties) under a common key.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.type = LONG, schema.fields.1.name = test2
- *
- * The arity of each propertyValue must match the arity of propertyKeys.
- */
- private def putIndexedFixedProperties(
- key: String,
- propertyKeys: Seq[String],
- propertyValues: Seq[Seq[String]])
- : Unit = {
- checkNotNull(key)
- checkNotNull(propertyValues)
- propertyValues.zipWithIndex.foreach { case (values, idx) =>
- if (values.lengthCompare(propertyKeys.size) != 0) {
- throw new ValidationException("Values must have same arity as keys.")
- }
- values.zipWithIndex.foreach { case (value, keyIdx) =>
- put(s"$key.$idx.${propertyKeys(keyIdx)}", value)
- }
- }
- }
-
- /**
- * Adds an indexed mapping of properties under a common key.
- *
- * For example:
- *
- * schema.fields.0.type = INT, schema.fields.0.name = test
- * schema.fields.1.name = test2
- *
- * The arity of the propertySets can differ.
- */
- private def putIndexedVariableProperties(
- key: String,
- propertySets: Seq[Map[String, String]])
- : Unit = {
- checkNotNull(key)
- checkNotNull(propertySets)
- propertySets.zipWithIndex.foreach { case (propertySet, idx) =>
- propertySet.foreach { case (k, v) =>
- put(s"$key.$idx.$k", v)
- }
- }
- }
-
- /**
- * Validates a property by first parsing the string value to a comparable object.
- * The boundaries are inclusive.
- */
- private def validateComparable[T <: Comparable[T]](
- key: String,
- isOptional: Boolean,
- min: T,
- max: T,
- typeName: String,
- parseFunction: String => T)
- : Unit = {
- if (!properties.contains(key)) {
- if (!isOptional) {
- throw new ValidationException(s"Could not find required property '$key'.")
- }
- } else {
- try {
- val value = parseFunction(properties(key))
-
- if (value.compareTo(min) < 0 || value.compareTo(max) > 0) {
- throw new ValidationException(s"Property '$key' must be a $typeName" +
- s" value between $min and $max but was: ${properties(key)}")
- }
- } catch {
- case _: NumberFormatException | _: IllegalArgumentException =>
- throw new ValidationException(
- s"Property '$key' must be a $typeName value but was: ${properties(key)}")
- }
- }
- }
-
- /**
- * Validates a property by first parsing the string value to a comparable object.
- * The boundaries are inclusive.
- */
- private def validateComparable[T <: Comparable[T]](
- key: String,
- isOptional: Boolean,
- min: T,
- max: T,
- parseFunction: String => T)
- : Unit = {
-
- validateComparable(
- key,
- isOptional,
- min,
- max,
- min.getClass.getSimpleName,
- parseFunction)
- }
-
- private def extractMaxIndex(key: String, suffixPattern: String = ""): Int = {
- // extract index and property keys
- val escapedKey = Pattern.quote(key)
- val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)$suffixPattern")
- val indexes = properties.keys.flatMap { k =>
- val matcher = pattern.matcher(k)
- if (matcher.find()) {
- Some(JInt.parseInt(matcher.group(1)))
- } else {
- None
- }
- }
-
- // determine max index
- indexes.reduceOption(_ max _).getOrElse(-1)
- }
-}
-
-object DescriptorProperties {
-
- private val emptyConsumer: Consumer[String] = new Consumer[String] {
- override def accept(t: String): Unit = {
- // nothing to do
- }
- }
-
- val TYPE: String = "type"
- val NAME: String = "name"
-
- // the string representation should be equal to SqlTypeName
- def normalizeTypeInfo(typeInfo: TypeInformation[_]): String = {
- checkNotNull(typeInfo)
- TypeStringUtils.writeTypeInfo(typeInfo)
- }
-
- def normalizeTableSchema(schema: TableSchema): Seq[(String, String)] = {
- schema.getFieldNames.zip(schema.getFieldTypes).map { case (n, t) =>
- (n, normalizeTypeInfo(t))
- }
- }
-
- def serialize(obj: Serializable): String = {
- // test public accessibility
- val error = InstantiationUtil.checkForInstantiationError(obj.getClass)
- if (error != null) {
- throw new ValidationException(s"Class '${obj.getClass.getName}' is not supported: $error")
- }
- try {
- val byteArray = InstantiationUtil.serializeObject(obj)
- Base64.encodeBase64URLSafeString(byteArray)
- } catch {
- case e: Exception =>
- throw new ValidationException(
- s"Unable to serialize class '${obj.getClass.getCanonicalName}'.", e)
- }
- }
-
- def deserialize[T](data: String, expected: Class[T]): T = {
- try {
- val byteData = Base64.decodeBase64(data)
- val obj = InstantiationUtil.deserializeObject[T](
- byteData,
- Thread.currentThread.getContextClassLoader)
- if (!expected.isAssignableFrom(obj.getClass)) {
- throw new ValidationException(
- s"Serialized data contains an object of unexpected type. " +
- s"Expected '${expected.getName}' but was '${obj.getClass.getName}'")
- }
- obj
- } catch {
- case e: Exception =>
- throw new ValidationException(s"Could not deserialize data: '$data'", e)
- }
- }
-
- def toString(keyOrValue: String): String = {
- StringEscapeUtils.escapeJava(keyOrValue).replace("\\/", "/") // '/' must not be escaped
- }
-
- def toString(key: String, value: String): String = {
- toString(key) + "=" + toString(value)
- }
-
- def toString(kv: Map[String, String]): String = {
- kv.map(e => DescriptorProperties.toString(e._1, e._2))
- .toSeq
- .sorted
- .mkString("\n")
- }
-
- def toJavaMap(descriptor: Descriptor): util.Map[String, String] = {
- val descriptorProperties = new DescriptorProperties()
- descriptor.addProperties(descriptorProperties)
- descriptorProperties.asMap
- }
-
- // the following methods help for Scala <-> Java interfaces
- // most of these methods are not necessary once we upgraded to Scala 2.12
-
- def toJava[T](option: Option[T]): Optional[T] = option match {
- case Some(v) => Optional.of(v)
- case None => Optional.empty()
- }
-
- def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T]))
-
- def toJava[T](func: (T) => Unit): Consumer[T] = new Consumer[T] {
- override def accept(t: T): Unit = {
- func.apply(t)
- }
- }
-
- def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] = new BiConsumer[K, V] {
- override def accept(k: K, v: V): Unit = {
- func.apply(k ,v)
- }
- }
-
- def toJava[I, O](func: (I) => O): Function[I, O] = new Function[I, O] {
- override def apply(in: I): O = {
- func.apply(in)
- }
- }
-
- def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
- new JTuple2[T0, T1](tuple._1, tuple._2)
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystemValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystemValidator.scala
index b065b5f..4d1b7de 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystemValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystemValidator.scala
@@ -28,8 +28,8 @@ class FileSystemValidator extends ConnectorDescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
super.validate(properties)
- properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE, isOptional = false)
- properties.validateString(CONNECTOR_PATH, isOptional = false, minLen = 1)
+ properties.validateValue(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE, false)
+ properties.validateString(CONNECTOR_PATH, false, 1)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
index 3d44c24..5c37c0a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
@@ -26,8 +26,8 @@ import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROP
class FormatDescriptorValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateString(FORMAT_TYPE, isOptional = false, minLen = 1)
- properties.validateInt(FORMAT_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateString(FORMAT_TYPE, false, 1)
+ properties.validateInt(FORMAT_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptorValidator.scala
index c119777..db8a1b5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FunctionDescriptorValidator.scala
@@ -19,8 +19,8 @@
package org.apache.flink.table.descriptors
import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.descriptors.DescriptorProperties.toJava
import org.apache.flink.table.descriptors.FunctionDescriptorValidator.FROM
+import org.apache.flink.table.util.JavaScalaConversionUtil.toJava
import scala.collection.JavaConverters._
@@ -39,7 +39,7 @@ class FunctionDescriptorValidator extends DescriptorValidator {
if (properties.containsKey(FROM)) {
properties.validateEnum(
FROM,
- isOptional = false,
+ false,
Map(
FunctionDescriptorValidator.FROM_VALUE_CLASS -> toJava(classValidation)
).asJava
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
index f567131..ba488d8 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/LiteralValueValidator.scala
@@ -46,22 +46,22 @@ class LiteralValueValidator(keyPrefix: String) extends HierarchyDescriptorValida
val typeKey = s"$keyPrefix${LiteralValueValidator.TYPE}"
- properties.validateType(typeKey, isOptional = true, requireRow = false)
+ properties.validateType(typeKey, true, false)
// explicit type
if (properties.containsKey(typeKey)) {
val valueKey = s"$keyPrefix${LiteralValueValidator.VALUE}"
val typeInfo = properties.getType(typeKey)
typeInfo match {
- case Types.DECIMAL => properties.validateBigDecimal(valueKey, isOptional = false)
- case Types.BOOLEAN => properties.validateBoolean(valueKey, isOptional = false)
- case Types.BYTE => properties.validateByte(valueKey, isOptional = false)
- case Types.DOUBLE => properties.validateDouble(valueKey, isOptional = false)
- case Types.FLOAT => properties.validateFloat(valueKey, isOptional = false)
- case Types.INT => properties.validateInt(valueKey, isOptional = false)
- case Types.LONG => properties.validateLong(valueKey, isOptional = false)
- case Types.SHORT => properties.validateShort(valueKey, isOptional = false)
- case Types.STRING => properties.validateString(valueKey, isOptional = false)
+ case Types.DECIMAL => properties.validateBigDecimal(valueKey, false)
+ case Types.BOOLEAN => properties.validateBoolean(valueKey, false)
+ case Types.BYTE => properties.validateByte(valueKey, false)
+ case Types.DOUBLE => properties.validateDouble(valueKey, false)
+ case Types.FLOAT => properties.validateFloat(valueKey, false)
+ case Types.INT => properties.validateInt(valueKey, false)
+ case Types.LONG => properties.validateLong(valueKey, false)
+ case Types.SHORT => properties.validateShort(valueKey, false)
+ case Types.STRING => properties.validateString(valueKey, false)
case _ => throw new TableException(s"Unsupported type '$typeInfo'.")
}
}
@@ -72,7 +72,7 @@ class LiteralValueValidator(keyPrefix: String) extends HierarchyDescriptorValida
throw new ValidationException(
"Literal values with implicit type must not exist in the top level of a hierarchy.")
}
- properties.validateString(keyPrefix.substring(0, keyPrefix.length - 1), isOptional = false)
+ properties.validateString(keyPrefix.substring(0, keyPrefix.length - 1), false)
}
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
index 6631e22..9c30872 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
@@ -26,10 +26,10 @@ import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, M
class MetadataValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(METADATA_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
- properties.validateString(METADATA_COMMENT, isOptional = true)
- properties.validateLong(METADATA_CREATION_TIME, isOptional = true)
- properties.validateLong(METADATA_LAST_ACCESS_TIME, isOptional = true)
+ properties.validateInt(METADATA_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
+ properties.validateString(METADATA_COMMENT, true)
+ properties.validateLong(METADATA_CREATION_TIME, true)
+ properties.validateLong(METADATA_LAST_ACCESS_TIME, true)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
index 4bd51c0..8b4cacd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
@@ -18,10 +18,11 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.DescriptorProperties.{serialize, toJava}
import org.apache.flink.table.descriptors.RowtimeValidator._
import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor}
import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy}
+import org.apache.flink.table.util.JavaScalaConversionUtil.toJava
+import org.apache.flink.table.utils.EncodingUtils
import scala.collection.JavaConverters._
@@ -37,20 +38,20 @@ class RowtimeValidator(
override def validate(properties: DescriptorProperties): Unit = {
val timestampExistingField = (_: String) => {
properties.validateString(
- prefix + ROWTIME_TIMESTAMPS_FROM, isOptional = false, minLen = 1)
+ prefix + ROWTIME_TIMESTAMPS_FROM, false, 1)
}
val timestampCustom = (_: String) => {
properties.validateString(
- prefix + ROWTIME_TIMESTAMPS_CLASS, isOptional = false, minLen = 1)
+ prefix + ROWTIME_TIMESTAMPS_CLASS, false, 1)
properties.validateString(
- prefix + ROWTIME_TIMESTAMPS_SERIALIZED, isOptional = false, minLen = 1)
+ prefix + ROWTIME_TIMESTAMPS_SERIALIZED, false, 1)
}
val timestampsValidation = if (supportsSourceTimestamps) {
Map(
ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD -> toJava(timestampExistingField),
- ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(),
+ ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> DescriptorProperties.noValidation(),
ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM -> toJava(timestampCustom))
} else {
Map(
@@ -60,38 +61,38 @@ class RowtimeValidator(
properties.validateEnum(
prefix + ROWTIME_TIMESTAMPS_TYPE,
- isOptional = false,
+ false,
timestampsValidation.asJava
)
val watermarkPeriodicBounded = (_: String) => {
properties.validateLong(
- prefix + ROWTIME_WATERMARKS_DELAY, isOptional = false, min = 0)
+ prefix + ROWTIME_WATERMARKS_DELAY, false, 0)
}
val watermarkCustom = (_: String) => {
properties.validateString(
- prefix + ROWTIME_WATERMARKS_CLASS, isOptional = false, minLen = 1)
+ prefix + ROWTIME_WATERMARKS_CLASS, false, 1)
properties.validateString(
- prefix + ROWTIME_WATERMARKS_SERIALIZED, isOptional = false, minLen = 1)
+ prefix + ROWTIME_WATERMARKS_SERIALIZED, false, 1)
}
val watermarksValidation = if (supportsSourceWatermarks) {
Map(
- ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> properties.noValidation(),
+ ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> DescriptorProperties.noValidation(),
ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED -> toJava(watermarkPeriodicBounded),
- ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(),
+ ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE -> DescriptorProperties.noValidation(),
ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM -> toJava(watermarkCustom))
} else {
Map(
- ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> properties.noValidation(),
+ ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> DescriptorProperties.noValidation(),
ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED -> toJava(watermarkPeriodicBounded),
ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM -> toJava(watermarkCustom))
}
properties.validateEnum(
prefix + ROWTIME_WATERMARKS_TYPE,
- isOptional = false,
+ false,
watermarksValidation.asJava
)
}
@@ -134,7 +135,7 @@ object RowtimeValidator {
Map(
ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM,
ROWTIME_TIMESTAMPS_CLASS -> extractor.getClass.getName,
- ROWTIME_TIMESTAMPS_SERIALIZED -> serialize(extractor))
+ ROWTIME_TIMESTAMPS_SERIALIZED -> EncodingUtils.encodeObjectToString(extractor))
}
def normalizeWatermarkStrategy(strategy: WatermarkStrategy): Map[String, String] =
@@ -155,7 +156,7 @@ object RowtimeValidator {
Map(
ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM,
ROWTIME_WATERMARKS_CLASS -> strategy.getClass.getName,
- ROWTIME_WATERMARKS_SERIALIZED -> serialize(strategy))
+ ROWTIME_WATERMARKS_SERIALIZED -> EncodingUtils.encodeObjectToString(strategy))
}
def getRowtimeComponents(properties: DescriptorProperties, prefix: String)
@@ -179,7 +180,7 @@ object RowtimeValidator {
val clazz = properties.getClass(
prefix + ROWTIME_TIMESTAMPS_CLASS,
classOf[TimestampExtractor])
- DescriptorProperties.deserialize(
+ EncodingUtils.decodeStringToObject(
properties.getString(prefix + ROWTIME_TIMESTAMPS_SERIALIZED),
clazz)
}
@@ -202,7 +203,7 @@ object RowtimeValidator {
val clazz = properties.getClass(
prefix + ROWTIME_WATERMARKS_CLASS,
classOf[WatermarkStrategy])
- DescriptorProperties.deserialize(
+ EncodingUtils.decodeStringToObject(
properties.getString(prefix + ROWTIME_WATERMARKS_SERIALIZED),
clazz)
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
index fcbb2c7..f4224b9 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
@@ -20,11 +20,11 @@ package org.apache.flink.table.descriptors
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.{TableSchema, ValidationException}
-import org.apache.flink.table.descriptors.DescriptorProperties.{normalizeTableSchema, normalizeTypeInfo}
import org.apache.flink.table.descriptors.SchemaValidator._
+import org.apache.flink.table.utils.TypeStringUtils
-import scala.collection.mutable
import scala.collection.JavaConverters._
+import scala.collection.mutable
/**
* Describes a schema of a table.
@@ -48,8 +48,8 @@ class Schema extends Descriptor {
def schema(schema: TableSchema): Schema = {
tableSchema.clear()
lastField = None
- normalizeTableSchema(schema).foreach {
- case (n, t) => field(n, t)
+ schema.getFieldNames.zip(schema.getFieldTypes).foreach { case (n, t) =>
+ field(n, t)
}
this
}
@@ -63,7 +63,7 @@ class Schema extends Descriptor {
* @param fieldType the type information of the field
*/
def field(fieldName: String, fieldType: TypeInformation[_]): Schema = {
- field(fieldName, normalizeTypeInfo(fieldType))
+ field(fieldName, TypeStringUtils.writeTypeInfo(fieldType))
this
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
index f6279bb..8c8230a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
@@ -24,10 +24,10 @@ import java.util.Optional
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeutils.CompositeType
import org.apache.flink.table.api.{TableException, TableSchema, ValidationException}
-import org.apache.flink.table.descriptors.DescriptorProperties.{toJava, toScala}
import org.apache.flink.table.descriptors.RowtimeValidator._
import org.apache.flink.table.descriptors.SchemaValidator._
import org.apache.flink.table.sources.RowtimeAttributeDescriptor
+import org.apache.flink.table.util.JavaScalaConversionUtil.{toJava, toScala}
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -54,11 +54,11 @@ class SchemaValidator(
for (i <- 0 until Math.max(names.size, types.size)) {
properties
- .validateString(s"$SCHEMA.$i.$SCHEMA_NAME", isOptional = false, minLen = 1)
+ .validateString(s"$SCHEMA.$i.$SCHEMA_NAME", false, 1)
properties
- .validateType(s"$SCHEMA.$i.$SCHEMA_TYPE", requireRow = false, isOptional = false)
+ .validateType(s"$SCHEMA.$i.$SCHEMA_TYPE", false, false)
properties
- .validateString(s"$SCHEMA.$i.$SCHEMA_FROM", isOptional = true, minLen = 1)
+ .validateString(s"$SCHEMA.$i.$SCHEMA_FROM", true, 1)
// either proctime or rowtime
val proctime = s"$SCHEMA.$i.$SCHEMA_PROCTIME"
val rowtime = s"$SCHEMA.$i.$ROWTIME"
@@ -73,7 +73,7 @@ class SchemaValidator(
throw new ValidationException("A proctime attribute must only be defined once.")
}
// check proctime
- properties.validateBoolean(proctime, isOptional = false)
+ properties.validateBoolean(proctime, false)
proctimeFound = properties.getBoolean(proctime)
// no rowtime
properties.validatePrefixExclusion(rowtime)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
index 691cb21..518f292 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
@@ -19,9 +19,9 @@
package org.apache.flink.table.descriptors
import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_PROPERTY_VERSION, STATISTICS_ROW_COUNT, validateColumnStats}
import org.apache.flink.table.plan.stats.ColumnStats
+import org.apache.flink.table.util.JavaScalaConversionUtil.toScala
import scala.collection.mutable
@@ -31,8 +31,8 @@ import scala.collection.mutable
class StatisticsValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(STATISTICS_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
- properties.validateLong(STATISTICS_ROW_COUNT, isOptional = true, min = 0)
+ properties.validateInt(STATISTICS_PROPERTY_VERSION, true, 0, Integer.MAX_VALUE)
+ properties.validateLong(STATISTICS_ROW_COUNT, true, 0)
validateColumnStats(properties, STATISTICS_COLUMNS)
}
}
@@ -84,13 +84,13 @@ object StatisticsValidator {
val columnCount = properties.getIndexedProperty(key, NAME).size
for (i <- 0 until columnCount) {
- properties.validateString(s"$key.$i.$NAME", isOptional = false, minLen = 1)
- properties.validateLong(s"$key.$i.$DISTINCT_COUNT", isOptional = true, min = 0L)
- properties.validateLong(s"$key.$i.$NULL_COUNT", isOptional = true, min = 0L)
- properties.validateDouble(s"$key.$i.$AVG_LENGTH", isOptional = true, min = 0.0)
- properties.validateInt(s"$key.$i.$MAX_LENGTH", isOptional = true, min = 0)
- properties.validateDouble(s"$key.$i.$MAX_VALUE", isOptional = true, min = 0.0)
- properties.validateDouble(s"$key.$i.$MIN_VALUE", isOptional = true, min = 0.0)
+ properties.validateString(s"$key.$i.$NAME", false, 1)
+ properties.validateLong(s"$key.$i.$DISTINCT_COUNT", true, 0L)
+ properties.validateLong(s"$key.$i.$NULL_COUNT", true, 0L)
+ properties.validateDouble(s"$key.$i.$AVG_LENGTH", true, 0.0)
+ properties.validateInt(s"$key.$i.$MAX_LENGTH", true, 0)
+ properties.validateDouble(s"$key.$i.$MAX_VALUE", true, 0.0)
+ properties.validateDouble(s"$key.$i.$MIN_VALUE", true, 0.0)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptorValidator.scala
index 8d60f89..7412e5b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableDescriptorValidator.scala
@@ -44,7 +44,7 @@ class StreamTableDescriptorValidator(
}
properties.validateEnumValues(
UPDATE_MODE,
- isOptional = false,
+ false,
modeList
)
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
index 26b7c6d..da7c9cd 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryService.scala
@@ -26,7 +26,7 @@ import org.apache.flink.table.descriptors.FormatDescriptorValidator._
import org.apache.flink.table.descriptors.MetadataValidator._
import org.apache.flink.table.descriptors.StatisticsValidator._
import org.apache.flink.table.descriptors._
-import org.apache.flink.table.util.Logging
+import org.apache.flink.table.util.{JavaScalaConversionUtil, Logging}
import org.apache.flink.util.Preconditions
import _root_.scala.collection.JavaConverters._
@@ -50,7 +50,7 @@ object TableFactoryService extends Logging {
def find[T](factoryClass: Class[T], descriptor: Descriptor): T = {
Preconditions.checkNotNull(descriptor)
- findInternal(factoryClass, DescriptorProperties.toJavaMap(descriptor), None)
+ findInternal(factoryClass, JavaScalaConversionUtil.toJavaMap(descriptor), None)
}
/**
@@ -66,7 +66,7 @@ object TableFactoryService extends Logging {
Preconditions.checkNotNull(descriptor)
Preconditions.checkNotNull(classLoader)
- findInternal(factoryClass, DescriptorProperties.toJavaMap(descriptor), Some(classLoader))
+ findInternal(factoryClass, JavaScalaConversionUtil.toJavaMap(descriptor), Some(classLoader))
}
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
index 9989ebc..1656ac7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/factories/TableFactoryUtil.scala
@@ -19,9 +19,10 @@
package org.apache.flink.table.factories
import org.apache.flink.table.api.{BatchTableEnvironment, StreamTableEnvironment, TableEnvironment, TableException}
-import org.apache.flink.table.descriptors.{Descriptor, DescriptorProperties}
+import org.apache.flink.table.descriptors.Descriptor
import org.apache.flink.table.sinks.TableSink
import org.apache.flink.table.sources.TableSource
+import org.apache.flink.table.util.JavaScalaConversionUtil
/**
* Utility for dealing with [[TableFactory]] using the [[TableFactoryService]].
@@ -36,7 +37,7 @@ object TableFactoryUtil {
descriptor: Descriptor)
: TableSource[T] = {
- val javaMap = DescriptorProperties.toJavaMap(descriptor)
+ val javaMap = JavaScalaConversionUtil.toJavaMap(descriptor)
tableEnvironment match {
case _: BatchTableEnvironment =>
@@ -62,7 +63,7 @@ object TableFactoryUtil {
descriptor: Descriptor)
: TableSink[T] = {
- val javaMap = DescriptorProperties.toJavaMap(descriptor)
+ val javaMap = JavaScalaConversionUtil.toJavaMap(descriptor)
tableEnvironment match {
case _: BatchTableEnvironment =>
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
index 38cb197..5ee9f4e 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sinks/CsvTableSinkFactoryBase.scala
@@ -23,12 +23,12 @@ import java.util
import org.apache.flink.table.api.TableException
import org.apache.flink.table.descriptors.ConnectorDescriptorValidator._
import org.apache.flink.table.descriptors.CsvValidator._
-import org.apache.flink.table.descriptors.DescriptorProperties._
import org.apache.flink.table.descriptors.FileSystemValidator._
import org.apache.flink.table.descriptors.FormatDescriptorValidator._
import org.apache.flink.table.descriptors.SchemaValidator._
import org.apache.flink.table.descriptors._
import org.apache.flink.table.factories.TableFactory
+import org.apache.flink.table.util.JavaScalaConversionUtil.toScala
/**
* Factory base for creating configured instances of [[CsvTableSink]].
@@ -49,13 +49,13 @@ abstract class CsvTableSinkFactoryBase extends TableFactory {
// connector
properties.add(CONNECTOR_PATH)
// format
- properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TYPE}")
- properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.NAME}")
+ properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TABLE_SCHEMA_TYPE}")
+ properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TABLE_SCHEMA_NAME}")
properties.add(FORMAT_FIELD_DELIMITER)
properties.add(CONNECTOR_PATH)
// schema
- properties.add(s"$SCHEMA.#.${DescriptorProperties.TYPE}")
- properties.add(s"$SCHEMA.#.${DescriptorProperties.NAME}")
+ properties.add(s"$SCHEMA.#.${DescriptorProperties.TABLE_SCHEMA_TYPE}")
+ properties.add(s"$SCHEMA.#.${DescriptorProperties.TABLE_SCHEMA_NAME}")
properties
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
index c2cf8f5..4a257bc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactoryBase.scala
@@ -23,12 +23,12 @@ import java.util
import org.apache.flink.table.api.TableException
import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_PROPERTY_VERSION, CONNECTOR_TYPE}
import org.apache.flink.table.descriptors.CsvValidator._
-import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, CONNECTOR_TYPE_VALUE}
import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE}
import org.apache.flink.table.descriptors.SchemaValidator.SCHEMA
import org.apache.flink.table.descriptors._
import org.apache.flink.table.factories.TableFactory
+import org.apache.flink.table.util.JavaScalaConversionUtil.toScala
/**
* Factory base for creating configured instances of [[CsvTableSource]].
@@ -49,8 +49,8 @@ abstract class CsvTableSourceFactoryBase extends TableFactory {
// connector
properties.add(CONNECTOR_PATH)
// format
- properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TYPE}")
- properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.NAME}")
+ properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TABLE_SCHEMA_TYPE}")
+ properties.add(s"$FORMAT_FIELDS.#.${DescriptorProperties.TABLE_SCHEMA_NAME}")
properties.add(FORMAT_FIELD_DELIMITER)
properties.add(FORMAT_LINE_DELIMITER)
properties.add(FORMAT_QUOTE_CHARACTER)
@@ -59,8 +59,8 @@ abstract class CsvTableSourceFactoryBase extends TableFactory {
properties.add(FORMAT_IGNORE_PARSE_ERRORS)
properties.add(CONNECTOR_PATH)
// schema
- properties.add(s"$SCHEMA.#.${DescriptorProperties.TYPE}")
- properties.add(s"$SCHEMA.#.${DescriptorProperties.NAME}")
+ properties.add(s"$SCHEMA.#.${DescriptorProperties.TABLE_SCHEMA_TYPE}")
+ properties.add(s"$SCHEMA.#.${DescriptorProperties.TABLE_SCHEMA_NAME}")
properties
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
new file mode 100644
index 0000000..3a60dbc
--- /dev/null
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/util/JavaScalaConversionUtil.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.util
+
+import java.util
+import java.util.Optional
+import java.util.function.{BiConsumer, Consumer, Function}
+
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.table.descriptors.{Descriptor, DescriptorProperties}
+
+/**
+ * Utilities for interoperability between Scala and Java classes.
+ */
+object JavaScalaConversionUtil {
+
+ // most of these methods are not necessary once we upgraded to Scala 2.12
+
+ def toJava[T](option: Option[T]): Optional[T] = option match {
+ case Some(v) => Optional.of(v)
+ case None => Optional.empty()
+ }
+
+ def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T]))
+
+ def toJava[T](func: (T) => Unit): Consumer[T] = new Consumer[T] {
+ override def accept(t: T): Unit = {
+ func.apply(t)
+ }
+ }
+
+ def toJava[K, V](func: (K, V) => Unit): BiConsumer[K, V] = new BiConsumer[K, V] {
+ override def accept(k: K, v: V): Unit = {
+ func.apply(k ,v)
+ }
+ }
+
+ def toJava[I, O](func: (I) => O): Function[I, O] = new Function[I, O] {
+ override def apply(in: I): O = {
+ func.apply(in)
+ }
+ }
+
+ def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
+ new JTuple2[T0, T1](tuple._1, tuple._2)
+ }
+
+ def toJavaMap(descriptor: Descriptor): util.Map[String, String] = {
+ val descriptorProperties = new DescriptorProperties()
+ descriptor.addProperties(descriptorProperties)
+ descriptorProperties.asMap
+ }
+}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala
index a8b279d..b2a8ec9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorPropertiesTest.scala
@@ -22,7 +22,7 @@ import java.util
import java.util.Collections
import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.descriptors.DescriptorProperties.toJava
+import org.apache.flink.table.util.JavaScalaConversionUtil.toJava
import org.junit.Assert.assertEquals
import org.junit.Test
@@ -66,9 +66,9 @@ class DescriptorPropertiesTest {
def testArrayValues(): Unit = {
val properties = new DescriptorProperties()
- properties.unsafePut(s"$ARRAY_KEY.0", "12")
- properties.unsafePut(s"$ARRAY_KEY.1", "42")
- properties.unsafePut(s"$ARRAY_KEY.2", "66")
+ properties.putString(s"$ARRAY_KEY.0", "12")
+ properties.putString(s"$ARRAY_KEY.1", "42")
+ properties.putString(s"$ARRAY_KEY.2", "66")
testArrayValidation(properties, 1, Integer.MAX_VALUE)
@@ -82,7 +82,7 @@ class DescriptorPropertiesTest {
@Test
def testArraySingleValue(): Unit = {
val properties = new DescriptorProperties()
- properties.unsafePut(ARRAY_KEY, "12")
+ properties.putString(ARRAY_KEY, "12")
testArrayValidation(properties, 1, Integer.MAX_VALUE)
@@ -96,9 +96,9 @@ class DescriptorPropertiesTest {
@Test(expected = classOf[ValidationException])
def testArrayInvalidValues(): Unit = {
val properties = new DescriptorProperties()
- properties.unsafePut(s"$ARRAY_KEY.0", "12")
- properties.unsafePut(s"$ARRAY_KEY.1", "INVALID")
- properties.unsafePut(s"$ARRAY_KEY.2", "66")
+ properties.putString(s"$ARRAY_KEY.0", "12")
+ properties.putString(s"$ARRAY_KEY.1", "INVALID")
+ properties.putString(s"$ARRAY_KEY.2", "66")
testArrayValidation(properties, 1, Integer.MAX_VALUE)
}
@@ -106,7 +106,7 @@ class DescriptorPropertiesTest {
@Test(expected = classOf[ValidationException])
def testArrayInvalidSingleValue(): Unit = {
val properties = new DescriptorProperties()
- properties.unsafePut(ARRAY_KEY, "INVALID")
+ properties.putString(ARRAY_KEY, "INVALID")
testArrayValidation(properties, 1, Integer.MAX_VALUE)
}
@@ -156,7 +156,7 @@ class DescriptorPropertiesTest {
maxLength: Int)
: Unit = {
val validator: (String) => Unit = (key: String) => {
- properties.validateInt(key, isOptional = false)
+ properties.validateInt(key, false)
}
properties.validateArray(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
index 3f6426d..c062cd9 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util.Collections
+
import org.apache.flink.table.descriptors.StreamTableDescriptorValidator.{UPDATE_MODE, UPDATE_MODE_VALUE_APPEND, UPDATE_MODE_VALUE_RETRACT, UPDATE_MODE_VALUE_UPSERT}
import org.apache.flink.util.Preconditions
import org.junit.Assert.assertEquals
@@ -73,15 +75,15 @@ abstract class DescriptorTestBase {
invalidValue: String): Unit = {
val properties = new DescriptorProperties
descriptor.addProperties(properties)
- properties.unsafePut(property, invalidValue)
- validator().validate(properties)
+ val copy = properties.withoutKeys(Collections.singletonList(property))
+ copy.putString(property, invalidValue)
+ validator().validate(copy)
}
def removePropertyAndVerify(descriptor: Descriptor, removeProperty: String): Unit = {
val properties = new DescriptorProperties
descriptor.addProperties(properties)
- properties.unsafeRemove(removeProperty)
- validator().validate(properties)
+ validator().validate(properties.withoutKeys(Collections.singletonList(removeProperty)))
}
}