You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2018/02/27 19:25:16 UTC
[1/5] flink git commit: [FLINK-8538] [table] Improve unified table
sources
Repository: flink
Updated Branches:
refs/heads/release-1.5 a269f8519 -> db2c510fb
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala
index 15cf13b..85778ca 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/CsvTest.scala
@@ -18,16 +18,36 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.typeutils.TypeExtractor
import org.apache.flink.table.api.{TableSchema, Types, ValidationException}
import org.junit.Test
+import scala.collection.JavaConverters._
+
class CsvTest extends DescriptorTestBase {
- @Test
- def testCsv(): Unit = {
- val desc = Csv()
+ @Test(expected = classOf[ValidationException])
+ def testInvalidType(): Unit = {
+ addPropertyAndVerify(descriptors().get(0), "format.fields.0.type", "WHATEVER")
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidField(): Unit = {
+ addPropertyAndVerify(descriptors().get(0), "format.fields.10.name", "WHATEVER")
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testInvalidQuoteCharacter(): Unit = {
+ addPropertyAndVerify(descriptors().get(0), "format.quote-character", "qq")
+ }
+
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ val desc1 = Csv()
.field("field1", "STRING")
.field("field2", Types.SQL_TIMESTAMP)
.field("field3", TypeExtractor.createTypeInfo(classOf[Class[_]]))
@@ -35,9 +55,21 @@ class CsvTest extends DescriptorTestBase {
Array[String]("test", "row"),
Array[TypeInformation[_]](Types.INT, Types.STRING)))
.lineDelimiter("^")
- val expected = Seq(
+
+ val desc2 = Csv()
+ .schema(new TableSchema(
+ Array[String]("test", "row"),
+ Array[TypeInformation[_]](Types.INT, Types.STRING)))
+ .quoteCharacter('#')
+ .ignoreFirstLine()
+
+ util.Arrays.asList(desc1, desc2)
+ }
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val props1 = Map(
"format.type" -> "csv",
- "format.version" -> "1",
+ "format.property-version" -> "1",
"format.fields.0.name" -> "field1",
"format.fields.0.type" -> "STRING",
"format.fields.1.name" -> "field2",
@@ -47,53 +79,18 @@ class CsvTest extends DescriptorTestBase {
"format.fields.3.name" -> "field4",
"format.fields.3.type" -> "ROW(test INT, row VARCHAR)",
"format.line-delimiter" -> "^")
- verifyProperties(desc, expected)
- }
- @Test
- def testCsvTableSchema(): Unit = {
- val desc = Csv()
- .schema(new TableSchema(
- Array[String]("test", "row"),
- Array[TypeInformation[_]](Types.INT, Types.STRING)))
- .quoteCharacter('#')
- .ignoreFirstLine()
- val expected = Seq(
+ val props2 = Map(
"format.type" -> "csv",
- "format.version" -> "1",
+ "format.property-version" -> "1",
"format.fields.0.name" -> "test",
"format.fields.0.type" -> "INT",
"format.fields.1.name" -> "row",
"format.fields.1.type" -> "VARCHAR",
"format.quote-character" -> "#",
"format.ignore-first-line" -> "true")
- verifyProperties(desc, expected)
- }
- @Test(expected = classOf[ValidationException])
- def testInvalidType(): Unit = {
- verifyInvalidProperty("format.fields.0.type", "WHATEVER")
- }
-
- @Test(expected = classOf[ValidationException])
- def testInvalidField(): Unit = {
- verifyInvalidProperty("format.fields.10.name", "WHATEVER")
- }
-
- @Test(expected = classOf[ValidationException])
- def testInvalidQuoteCharacter(): Unit = {
- verifyInvalidProperty("format.quote-character", "qq")
- }
-
- override def descriptor(): Descriptor = {
- Csv()
- .field("field1", "STRING")
- .field("field2", Types.SQL_TIMESTAMP)
- .field("field3", TypeExtractor.createTypeInfo(classOf[Class[_]]))
- .field("field4", Types.ROW(
- Array[String]("test", "row"),
- Array[TypeInformation[_]](Types.INT, Types.STRING)))
- .lineDelimiter("^")
+ util.Arrays.asList(props1.asJava, props2.asJava)
}
override def validator(): DescriptorValidator = {
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
index 3a59c9b..7a98b0b 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/DescriptorTestBase.scala
@@ -18,37 +18,84 @@
package org.apache.flink.table.descriptors
+import org.apache.flink.util.Preconditions
import org.junit.Assert.assertEquals
+import org.junit.Test
+
+import scala.collection.JavaConverters._
abstract class DescriptorTestBase {
/**
- * Returns a valid descriptor.
+ * Returns a set of valid descriptors.
+ * This method is implemented in both Scala and Java.
*/
- def descriptor(): Descriptor
+ def descriptors(): java.util.List[Descriptor]
/**
- * Returns a validator that can validate this descriptor.
+ * Returns a set of properties for each valid descriptor.
+ * This code is implemented in both Scala and Java.
+ */
+ def properties(): java.util.List[java.util.Map[String, String]]
+
+ /**
+ * Returns a validator that can validate all valid descriptors.
*/
def validator(): DescriptorValidator
- def verifyProperties(descriptor: Descriptor, expected: Seq[(String, String)]): Unit = {
+ @Test
+ def testValidation(): Unit = {
+ val d = descriptors().asScala
+ val p = properties().asScala
+
+ Preconditions.checkArgument(d.length == p.length)
+
+ d.zip(p).foreach { case (desc, props) =>
+ verifyProperties(desc, props.asScala.toMap)
+ }
+ }
+
+ def verifyProperties(descriptor: Descriptor, expected: Map[String, String]): Unit = {
val normProps = new DescriptorProperties
descriptor.addProperties(normProps)
- assertEquals(expected.toMap, normProps.asMap)
+
+ // test produced properties
+ assertEquals(expected, normProps.asMap.asScala.toMap)
+
+ // test validation logic
+ validator().validate(normProps)
}
- def verifyInvalidProperty(property: String, invalidValue: String): Unit = {
+ def addPropertyAndVerify(
+ descriptor: Descriptor,
+ property: String,
+ invalidValue: String): Unit = {
val properties = new DescriptorProperties
- descriptor().addProperties(properties)
+ descriptor.addProperties(properties)
properties.unsafePut(property, invalidValue)
validator().validate(properties)
}
- def verifyMissingProperty(removeProperty: String): Unit = {
+ def removePropertyAndVerify(descriptor: Descriptor, removeProperty: String): Unit = {
val properties = new DescriptorProperties
- descriptor().addProperties(properties)
+ descriptor.addProperties(properties)
properties.unsafeRemove(removeProperty)
validator().validate(properties)
}
}
+
+class TestTableSourceDescriptor(connector: ConnectorDescriptor)
+ extends TableSourceDescriptor {
+
+ this.connectorDescriptor = Some(connector)
+
+ def addFormat(format: FormatDescriptor): TestTableSourceDescriptor = {
+ this.formatDescriptor = Some(format)
+ this
+ }
+
+ def addSchema(schema: Schema): TestTableSourceDescriptor = {
+ this.schemaDescriptor = Some(schema)
+ this
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala
index 3452e8d..1162694 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/FileSystemTest.scala
@@ -18,36 +18,41 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.api.ValidationException
import org.junit.Test
-class FileSystemTest extends DescriptorTestBase {
+import scala.collection.JavaConverters._
- @Test
- def testFileSystem(): Unit = {
- val desc = FileSystem().path("/myfile")
- val expected = Seq(
- "connector.type" -> "filesystem",
- "connector.version" -> "1",
- "connector.path" -> "/myfile")
- verifyProperties(desc, expected)
- }
+class FileSystemTest extends DescriptorTestBase {
@Test(expected = classOf[ValidationException])
def testInvalidPath(): Unit = {
- verifyInvalidProperty("connector.path", "")
+ addPropertyAndVerify(descriptors().get(0), "connector.path", "")
}
@Test(expected = classOf[ValidationException])
def testMissingPath(): Unit = {
- verifyMissingProperty("connector.path")
+ removePropertyAndVerify(descriptors().get(0), "connector.path")
}
- override def descriptor(): Descriptor = {
- FileSystem().path("/myfile")
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ util.Arrays.asList(FileSystem().path("/myfile"))
}
override def validator(): DescriptorValidator = {
new FileSystemValidator()
}
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val desc = Map(
+ "connector.type" -> "filesystem",
+ "connector.property-version" -> "1",
+ "connector.path" -> "/myfile")
+
+ util.Arrays.asList(desc.asJava)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala
deleted file mode 100644
index 756ca23..0000000
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/JsonTest.scala
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.api.ValidationException
-import org.junit.Test
-
-class JsonTest extends DescriptorTestBase {
-
- @Test
- def testJson(): Unit = {
- val schema =
- """
- |{
- | "title": "Person",
- | "type": "object",
- | "properties": {
- | "firstName": {
- | "type": "string"
- | },
- | "lastName": {
- | "type": "string"
- | },
- | "age": {
- | "description": "Age in years",
- | "type": "integer",
- | "minimum": 0
- | }
- | },
- | "required": ["firstName", "lastName"]
- |}
- |""".stripMargin
- val desc = Json()
- .schema(schema)
- .failOnMissingField(true)
- val expected = Seq(
- "format.type" -> "json",
- "format.version" -> "1",
- "format.schema-string" -> schema,
- "format.fail-on-missing-field" -> "true")
- verifyProperties(desc, expected)
- }
-
- @Test(expected = classOf[ValidationException])
- def testInvalidMissingField(): Unit = {
- verifyInvalidProperty("format.fail-on-missing-field", "DDD")
- }
-
- @Test(expected = classOf[ValidationException])
- def testMissingSchema(): Unit = {
- verifyMissingProperty("format.schema-string")
- }
-
- override def descriptor(): Descriptor = {
- Json().schema("test")
- }
-
- override def validator(): DescriptorValidator = {
- new JsonValidator()
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala
index a1854ce..64965b0 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/MetadataTest.scala
@@ -18,38 +18,42 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.api.ValidationException
import org.junit.Test
-class MetadataTest extends DescriptorTestBase {
+import scala.collection.JavaConverters._
- @Test
- def testMetadata(): Unit = {
- val desc = Metadata()
- .comment("Some additional comment")
- .creationTime(123L)
- .lastAccessTime(12020202L)
- val expected = Seq(
- "metadata.comment" -> "Some additional comment",
- "metadata.creation-time" -> "123",
- "metadata.last-access-time" -> "12020202"
- )
- verifyProperties(desc, expected)
- }
+class MetadataTest extends DescriptorTestBase {
@Test(expected = classOf[ValidationException])
def testInvalidCreationTime(): Unit = {
- verifyInvalidProperty("metadata.creation-time", "dfghj")
+ addPropertyAndVerify(descriptors().get(0), "metadata.creation-time", "dfghj")
}
- override def descriptor(): Descriptor = {
- Metadata()
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ val desc = Metadata()
.comment("Some additional comment")
.creationTime(123L)
.lastAccessTime(12020202L)
+
+ util.Arrays.asList(desc)
}
override def validator(): DescriptorValidator = {
new MetadataValidator()
}
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val props = Map(
+ "metadata.comment" -> "Some additional comment",
+ "metadata.creation-time" -> "123",
+ "metadata.last-access-time" -> "12020202"
+ )
+
+ util.Arrays.asList(props.asJava)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
index 80050fc..7968b48 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/RowtimeTest.scala
@@ -18,6 +18,8 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.table.api.ValidationException
import org.apache.flink.table.descriptors.RowtimeTest.CustomAssigner
@@ -25,41 +27,58 @@ import org.apache.flink.table.sources.wmstrategies.PunctuatedWatermarkAssigner
import org.apache.flink.types.Row
import org.junit.Test
-class RowtimeTest extends DescriptorTestBase {
+import scala.collection.JavaConverters._
- @Test
- def testRowtime(): Unit = {
- val desc = Rowtime()
- .timestampsFromField("otherField")
- .watermarksPeriodicBounding(1000L)
- val expected = Seq(
- "rowtime.0.version" -> "1",
- "rowtime.0.timestamps.type" -> "from-field",
- "rowtime.0.timestamps.from" -> "otherField",
- "rowtime.0.watermarks.type" -> "periodic-bounding",
- "rowtime.0.watermarks.delay" -> "1000"
- )
- verifyProperties(desc, expected)
- }
+class RowtimeTest extends DescriptorTestBase {
@Test(expected = classOf[ValidationException])
def testInvalidWatermarkType(): Unit = {
- verifyInvalidProperty("rowtime.0.watermarks.type", "xxx")
+ addPropertyAndVerify(descriptors().get(0), "rowtime.watermarks.type", "xxx")
}
@Test(expected = classOf[ValidationException])
def testMissingWatermarkClass(): Unit = {
- verifyMissingProperty("rowtime.0.watermarks.class")
+ removePropertyAndVerify(descriptors().get(1), "rowtime.watermarks.class")
}
- override def descriptor(): Descriptor = {
- Rowtime()
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ val desc1 = Rowtime()
+ .timestampsFromField("otherField")
+ .watermarksPeriodicBounding(1000L)
+
+ val desc2 = Rowtime()
.timestampsFromSource()
.watermarksFromStrategy(new CustomAssigner())
+
+ util.Arrays.asList(desc1, desc2)
}
override def validator(): DescriptorValidator = {
- new RowtimeValidator("rowtime.0.")
+ new RowtimeValidator()
+ }
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val props1 = Map(
+ "rowtime.timestamps.type" -> "from-field",
+ "rowtime.timestamps.from" -> "otherField",
+ "rowtime.watermarks.type" -> "periodic-bounded",
+ "rowtime.watermarks.delay" -> "1000"
+ )
+
+ val props2 = Map(
+ "rowtime.timestamps.type" -> "from-source",
+ "rowtime.watermarks.type" -> "custom",
+ "rowtime.watermarks.class" -> "org.apache.flink.table.descriptors.RowtimeTest$CustomAssigner",
+ "rowtime.watermarks.serialized" -> ("rO0ABXNyAD1vcmcuYXBhY2hlLmZsaW5rLnRhYmxlLmRlc2NyaX" +
+ "B0b3JzLlJvd3RpbWVUZXN0JEN1c3RvbUFzc2lnbmVyeDcuDvfbu0kCAAB4cgBHb3JnLmFwYWNoZS5mbGluay" +
+ "50YWJsZS5zb3VyY2VzLndtc3RyYXRlZ2llcy5QdW5jdHVhdGVkV2F0ZXJtYXJrQXNzaWduZXKBUc57oaWu9A" +
+ "IAAHhyAD1vcmcuYXBhY2hlLmZsaW5rLnRhYmxlLnNvdXJjZXMud21zdHJhdGVnaWVzLldhdGVybWFya1N0cm" +
+ "F0ZWd5mB_uSxDZ8-MCAAB4cA")
+ )
+
+ util.Arrays.asList(props1.asJava, props2.asJava)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala
index f663a96..589ec4f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaTest.scala
@@ -18,21 +18,54 @@
package org.apache.flink.table.descriptors
+import java.util
+
import org.apache.flink.table.api.{Types, ValidationException}
import org.junit.Test
+import scala.collection.JavaConverters._
+
class SchemaTest extends DescriptorTestBase {
- @Test
- def testSchema(): Unit = {
- val desc = Schema()
+ @Test(expected = classOf[ValidationException])
+ def testInvalidType(): Unit = {
+ addPropertyAndVerify(
+ descriptors().get(0),
+ "schema.1.type", "dfghj")
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testBothRowtimeAndProctime(): Unit = {
+ addPropertyAndVerify(
+ descriptors().get(0),
+ "schema.2.rowtime.watermarks.type", "from-source")
+ }
+
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ val desc1 = Schema()
.field("myField", Types.BOOLEAN)
.field("otherField", "VARCHAR").from("csvField")
.field("p", Types.SQL_TIMESTAMP).proctime()
.field("r", Types.SQL_TIMESTAMP).rowtime(
Rowtime().timestampsFromSource().watermarksFromSource())
- val expected = Seq(
- "schema.version" -> "1",
+
+ val desc2 = Schema()
+ .field("myField", Types.BOOLEAN)
+ .field("otherField", "VARCHAR").from("csvField")
+ .field("p", Types.SQL_TIMESTAMP).proctime()
+ .field("r", Types.SQL_TIMESTAMP)
+
+ util.Arrays.asList(desc1, desc2)
+ }
+
+ override def validator(): DescriptorValidator = {
+ new SchemaValidator(isStreamEnvironment = true)
+ }
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val props1 = Map(
"schema.0.name" -> "myField",
"schema.0.type" -> "BOOLEAN",
"schema.1.name" -> "otherField",
@@ -43,34 +76,23 @@ class SchemaTest extends DescriptorTestBase {
"schema.2.proctime" -> "true",
"schema.3.name" -> "r",
"schema.3.type" -> "TIMESTAMP",
- "schema.3.rowtime.0.version" -> "1",
- "schema.3.rowtime.0.watermarks.type" -> "from-source",
- "schema.3.rowtime.0.timestamps.type" -> "from-source"
+ "schema.3.rowtime.watermarks.type" -> "from-source",
+ "schema.3.rowtime.timestamps.type" -> "from-source"
)
- verifyProperties(desc, expected)
- }
- @Test(expected = classOf[ValidationException])
- def testInvalidType(): Unit = {
- verifyInvalidProperty("schema.1.type", "dfghj")
- }
-
- @Test(expected = classOf[ValidationException])
- def testBothRowtimeAndProctime(): Unit = {
- verifyInvalidProperty("schema.2.rowtime.0.version", "1")
- verifyInvalidProperty("schema.2.rowtime.0.watermarks.type", "from-source")
- verifyInvalidProperty("schema.2.rowtime.0.timestamps.type", "from-source")
- }
-
- override def descriptor(): Descriptor = {
- Schema()
- .field("myField", Types.BOOLEAN)
- .field("otherField", "VARCHAR").from("csvField")
- .field("p", Types.SQL_TIMESTAMP).proctime()
- .field("r", Types.SQL_TIMESTAMP)
- }
+ val props2 = Map(
+ "schema.0.name" -> "myField",
+ "schema.0.type" -> "BOOLEAN",
+ "schema.1.name" -> "otherField",
+ "schema.1.type" -> "VARCHAR",
+ "schema.1.from" -> "csvField",
+ "schema.2.name" -> "p",
+ "schema.2.type" -> "TIMESTAMP",
+ "schema.2.proctime" -> "true",
+ "schema.3.name" -> "r",
+ "schema.3.type" -> "TIMESTAMP"
+ )
- override def validator(): DescriptorValidator = {
- new SchemaValidator(isStreamEnvironment = true)
+ util.Arrays.asList(props1.asJava, props2.asJava)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
new file mode 100644
index 0000000..ba05dff
--- /dev/null
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/SchemaValidatorTest.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors
+
+import java.util.Optional
+
+import org.apache.flink.table.api.{TableSchema, Types}
+import org.apache.flink.table.sources.tsextractors.StreamRecordTimestamp
+import org.apache.flink.table.sources.wmstrategies.PreserveWatermarks
+import org.junit.Assert.{assertEquals, assertTrue}
+import org.junit.Test
+
+import scala.collection.JavaConverters._
+
+/**
+ * Tests for [[SchemaValidator]].
+ */
+class SchemaValidatorTest {
+
+ @Test
+ def testSchema(): Unit = {
+ val desc1 = Schema()
+ .field("otherField", Types.STRING).from("csvField")
+ .field("abcField", Types.STRING)
+ .field("p", Types.SQL_TIMESTAMP).proctime()
+ .field("r", Types.SQL_TIMESTAMP).rowtime(
+ Rowtime().timestampsFromSource().watermarksFromSource())
+ val props = new DescriptorProperties()
+ desc1.addProperties(props)
+
+ val inputSchema = TableSchema.builder()
+ .field("csvField", Types.STRING)
+ .field("abcField", Types.STRING)
+ .field("myField", Types.BOOLEAN)
+ .build()
+
+ // test proctime
+ assertEquals(Optional.of("p"), SchemaValidator.deriveProctimeAttribute(props))
+
+ // test rowtime
+ val rowtime = SchemaValidator.deriveRowtimeAttributes(props).get(0)
+ assertEquals("r", rowtime.getAttributeName)
+ assertTrue(rowtime.getTimestampExtractor.isInstanceOf[StreamRecordTimestamp])
+ assertTrue(rowtime.getWatermarkStrategy.isInstanceOf[PreserveWatermarks])
+
+ // test field mapping
+ val expectedMapping = Map("otherField" -> "csvField", "abcField" -> "abcField").asJava
+ assertEquals(
+ expectedMapping,
+ SchemaValidator.deriveFieldMapping(props, Optional.of(inputSchema)))
+
+ // test field format
+ val formatSchema = SchemaValidator.deriveFormatFields(props)
+ val expectedFormatSchema = TableSchema.builder()
+ .field("csvField", Types.STRING) // aliased
+ .field("abcField", Types.STRING)
+ .build()
+ assertEquals(expectedFormatSchema, formatSchema)
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala
index 3b248b4..2def0c3 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/descriptors/StatisticsTest.scala
@@ -24,17 +24,44 @@ import org.apache.flink.table.api.ValidationException
import org.apache.flink.table.plan.stats.{ColumnStats, TableStats}
import org.junit.Test
+import scala.collection.JavaConverters._
+
class StatisticsTest extends DescriptorTestBase {
- @Test
- def testStatistics(): Unit = {
- val desc = Statistics()
+ @Test(expected = classOf[ValidationException])
+ def testInvalidRowCount(): Unit = {
+ addPropertyAndVerify(descriptors().get(0), "statistics.row-count", "abx")
+ }
+
+ @Test(expected = classOf[ValidationException])
+ def testMissingName(): Unit = {
+ removePropertyAndVerify(descriptors().get(0), "statistics.columns.0.name")
+ }
+
+ // ----------------------------------------------------------------------------------------------
+
+ override def descriptors(): util.List[Descriptor] = {
+ val desc1 = Statistics()
.rowCount(1000L)
.columnStats("a", ColumnStats(1L, 2L, 3.0, 4, 5, 6))
.columnAvgLength("b", 42.0)
.columnNullCount("a", 300)
- val expected = Seq(
- "statistics.version" -> "1",
+
+ val map = new util.HashMap[String, ColumnStats]()
+ map.put("a", ColumnStats(null, 2L, 3.0, null, 5, 6))
+ val desc2 = Statistics()
+ .tableStats(TableStats(32L, map))
+
+ util.Arrays.asList(desc1, desc2)
+ }
+
+ override def validator(): DescriptorValidator = {
+ new StatisticsValidator()
+ }
+
+ override def properties(): util.List[util.Map[String, String]] = {
+ val props1 = Map(
+ "statistics.property-version" -> "1",
"statistics.row-count" -> "1000",
"statistics.columns.0.name" -> "a",
"statistics.columns.0.distinct-count" -> "1",
@@ -46,17 +73,9 @@ class StatisticsTest extends DescriptorTestBase {
"statistics.columns.1.name" -> "b",
"statistics.columns.1.avg-length" -> "42.0"
)
- verifyProperties(desc, expected)
- }
- @Test
- def testStatisticsTableStats(): Unit = {
- val map = new util.HashMap[String, ColumnStats]()
- map.put("a", ColumnStats(null, 2L, 3.0, null, 5, 6))
- val desc = Statistics()
- .tableStats(TableStats(32L, map))
- val expected = Seq(
- "statistics.version" -> "1",
+ val props2 = Map(
+ "statistics.property-version" -> "1",
"statistics.row-count" -> "32",
"statistics.columns.0.name" -> "a",
"statistics.columns.0.null-count" -> "2",
@@ -64,28 +83,7 @@ class StatisticsTest extends DescriptorTestBase {
"statistics.columns.0.max-value" -> "5",
"statistics.columns.0.min-value" -> "6"
)
- verifyProperties(desc, expected)
- }
- @Test(expected = classOf[ValidationException])
- def testInvalidRowCount(): Unit = {
- verifyInvalidProperty("statistics.row-count", "abx")
- }
-
- @Test(expected = classOf[ValidationException])
- def testMissingName(): Unit = {
- verifyMissingProperty("statistics.columns.0.name")
- }
-
- override def descriptor(): Descriptor = {
- Statistics()
- .rowCount(1000L)
- .columnStats("a", ColumnStats(1L, 2L, 3.0, 4, 5, 6))
- .columnAvgLength("b", 42.0)
- .columnNullCount("a", 300)
- }
-
- override def validator(): DescriptorValidator = {
- new StatisticsValidator()
+ util.Arrays.asList(props1.asJava, props2.asJava)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala
index 5e9b5a2..279e9a4 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TableSourceFactoryServiceTest.scala
@@ -19,8 +19,8 @@
package org.apache.flink.table.sources
import org.apache.flink.table.api.{NoMatchingTableSourceException, TableException, ValidationException}
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION}
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION}
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION}
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -31,44 +31,44 @@ class TableSourceFactoryServiceTest {
@Test
def testValidProperties(): Unit = {
val props = properties()
- assertTrue(TableSourceFactoryService.findTableSourceFactory(props.toMap) != null)
+ assertTrue(TableSourceFactoryService.findAndCreateTableSource(props.toMap) != null)
}
@Test(expected = classOf[NoMatchingTableSourceException])
def testInvalidContext(): Unit = {
val props = properties()
props.put(CONNECTOR_TYPE, "FAIL")
- TableSourceFactoryService.findTableSourceFactory(props.toMap)
+ TableSourceFactoryService.findAndCreateTableSource(props.toMap)
}
@Test
def testDifferentContextVersion(): Unit = {
val props = properties()
- props.put(CONNECTOR_VERSION, "2")
+ props.put(CONNECTOR_PROPERTY_VERSION, "2")
// the table source should still be found
- assertTrue(TableSourceFactoryService.findTableSourceFactory(props.toMap) != null)
+ assertTrue(TableSourceFactoryService.findAndCreateTableSource(props.toMap) != null)
}
@Test(expected = classOf[ValidationException])
def testUnsupportedProperty(): Unit = {
val props = properties()
props.put("format.path_new", "/new/path")
- TableSourceFactoryService.findTableSourceFactory(props.toMap)
+ TableSourceFactoryService.findAndCreateTableSource(props.toMap)
}
@Test(expected = classOf[TableException])
def testFailingFactory(): Unit = {
val props = properties()
props.put("failing", "true")
- TableSourceFactoryService.findTableSourceFactory(props.toMap)
+ TableSourceFactoryService.findAndCreateTableSource(props.toMap)
}
private def properties(): mutable.Map[String, String] = {
val properties = mutable.Map[String, String]()
properties.put(CONNECTOR_TYPE, "test")
properties.put(FORMAT_TYPE, "test")
- properties.put(CONNECTOR_VERSION, "1")
- properties.put(FORMAT_VERSION, "1")
+ properties.put(CONNECTOR_PROPERTY_VERSION, "1")
+ properties.put(FORMAT_PROPERTY_VERSION, "1")
properties.put("format.path", "/path/to/target")
properties.put("schema.0.name", "a")
properties.put("schema.1.name", "b")
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala
index ae75f99..ee3d637 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/sources/TestTableSourceFactory.scala
@@ -22,8 +22,8 @@ import java.util
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.table.api.TableSchema
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION}
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION}
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION}
import org.apache.flink.types.Row
class TestTableSourceFactory extends TableSourceFactory[Row] {
@@ -32,8 +32,8 @@ class TestTableSourceFactory extends TableSourceFactory[Row] {
val context = new util.HashMap[String, String]()
context.put(CONNECTOR_TYPE, "test")
context.put(FORMAT_TYPE, "test")
- context.put(CONNECTOR_VERSION, "1")
- context.put(FORMAT_VERSION, "1")
+ context.put(CONNECTOR_PROPERTY_VERSION, "1")
+ context.put(FORMAT_PROPERTY_VERSION, "1")
context
}
[3/5] flink git commit: [FLINK-8538] [table] Improve unified table
sources
Posted by tw...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
index a3ca22f..3adc7c5 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
@@ -19,175 +19,97 @@
package org.apache.flink.table.descriptors;
import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
-import org.apache.flink.table.api.ValidationException;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-
-import scala.Function0;
-import scala.Tuple2;
-import scala.collection.JavaConversions;
-import scala.runtime.AbstractFunction0;
-import scala.runtime.BoxedUnit;
-
+import java.util.function.Consumer;
/**
* The validator for {@link Kafka}.
*/
public class KafkaValidator extends ConnectorDescriptorValidator {
- // fields
- public static final String CONNECTOR_TYPE_VALUE = "kafka";
- public static final String KAFKA_VERSION = "kafka.version";
- public static final String BOOTSTRAP_SERVERS = "bootstrap.servers";
- public static final String GROUP_ID = "group.id";
- public static final String TOPIC = "topic";
- public static final String STARTUP_MODE = "startup.mode";
- public static final String SPECIFIC_OFFSETS = "specific.offsets";
- public static final String TABLE_JSON_MAPPING = "table.json.mapping";
-
- public static final String PARTITION = "partition";
- public static final String OFFSET = "offset";
-
- public static final String TABLE_FIELD = "table.field";
- public static final String JSON_FIELD = "json.field";
-
- public static final String ZOOKEEPER_CONNECT = "zookeeper.connect"; // only required for 0.8
- // values
- public static final String KAFKA_VERSION_VALUE_08 = "0.8";
- public static final String KAFKA_VERSION_VALUE_09 = "0.9";
- public static final String KAFKA_VERSION_VALUE_010 = "0.10";
- public static final String KAFKA_VERSION_VALUE_011 = "0.11";
-
- public static final String STARTUP_MODE_VALUE_EARLIEST = "earliest-offset";
- public static final String STARTUP_MODE_VALUE_LATEST = "latest-offset";
- public static final String STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets";
- public static final String STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets";
-
- // utils
- public static Map<String, String> normalizeStartupMode(StartupMode startupMode) {
- Map<String, String> mapPair = new HashMap<>();
- switch (startupMode) {
- case EARLIEST:
- mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_EARLIEST);
- break;
- case LATEST:
- mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_LATEST);
- break;
- case GROUP_OFFSETS:
- mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_GROUP_OFFSETS);
- break;
- case SPECIFIC_OFFSETS:
- mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_SPECIFIC_OFFSETS);
- break;
- }
- return mapPair;
- }
+ public static final String CONNECTOR_TYPE_VALUE_KAFKA = "kafka";
+ public static final String CONNECTOR_VERSION_VALUE_08 = "0.8";
+ public static final String CONNECTOR_VERSION_VALUE_09 = "0.9";
+ public static final String CONNECTOR_VERSION_VALUE_010 = "0.10";
+ public static final String CONNECTOR_VERSION_VALUE_011 = "0.11";
+ public static final String CONNECTOR_TOPIC = "connector.topic";
+ public static final String CONNECTOR_STARTUP_MODE = "connector.startup-mode";
+ public static final String CONNECTOR_STARTUP_MODE_VALUE_EARLIEST = "earliest-offset";
+ public static final String CONNECTOR_STARTUP_MODE_VALUE_LATEST = "latest-offset";
+ public static final String CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets";
+ public static final String CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets";
+ public static final String CONNECTOR_SPECIFIC_OFFSETS = "connector.specific-offsets";
+ public static final String CONNECTOR_SPECIFIC_OFFSETS_PARTITION = "partition";
+ public static final String CONNECTOR_SPECIFIC_OFFSETS_OFFSET = "offset";
+ public static final String CONNECTOR_PROPERTIES = "connector.properties";
+ public static final String CONNECTOR_PROPERTIES_KEY = "key";
+ public static final String CONNECTOR_PROPERTIES_VALUE = "value";
@Override
public void validate(DescriptorProperties properties) {
super.validate(properties);
-
- AbstractFunction0<BoxedUnit> emptyValidator = new AbstractFunction0<BoxedUnit>() {
- @Override
- public BoxedUnit apply() {
- return BoxedUnit.UNIT;
- }
- };
-
- properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE, false);
-
- AbstractFunction0<BoxedUnit> version08Validator = new AbstractFunction0<BoxedUnit>() {
- @Override
- public BoxedUnit apply() {
- properties.validateString(ZOOKEEPER_CONNECT, false, 0, Integer.MAX_VALUE);
- return BoxedUnit.UNIT;
- }
- };
-
- Map<String, Function0<BoxedUnit>> versionValidatorMap = new HashMap<>();
- versionValidatorMap.put(KAFKA_VERSION_VALUE_08, version08Validator);
- versionValidatorMap.put(KAFKA_VERSION_VALUE_09, emptyValidator);
- versionValidatorMap.put(KAFKA_VERSION_VALUE_010, emptyValidator);
- versionValidatorMap.put(KAFKA_VERSION_VALUE_011, emptyValidator);
- properties.validateEnum(
- KAFKA_VERSION,
+ properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA, false);
+
+ final List<String> versions = Arrays.asList(
+ CONNECTOR_VERSION_VALUE_08,
+ CONNECTOR_VERSION_VALUE_09,
+ CONNECTOR_VERSION_VALUE_010,
+ CONNECTOR_VERSION_VALUE_011);
+ properties.validateEnumValues(CONNECTOR_VERSION(), false, versions);
+ properties.validateString(CONNECTOR_TOPIC, false, 1, Integer.MAX_VALUE);
+
+ final Map<String, Consumer<String>> specificOffsetValidators = new HashMap<>();
+ specificOffsetValidators.put(
+ CONNECTOR_SPECIFIC_OFFSETS_PARTITION,
+ (prefix) -> properties.validateInt(
+ prefix + CONNECTOR_SPECIFIC_OFFSETS_PARTITION,
false,
- toScalaImmutableMap(versionValidatorMap)
- );
-
- properties.validateString(BOOTSTRAP_SERVERS, false, 1, Integer.MAX_VALUE);
- properties.validateString(GROUP_ID, false, 1, Integer.MAX_VALUE);
- properties.validateString(TOPIC, false, 1, Integer.MAX_VALUE);
-
- AbstractFunction0<BoxedUnit> specificOffsetsValidator = new AbstractFunction0<BoxedUnit>() {
- @Override
- public BoxedUnit apply() {
- Map<String, String> partitions = JavaConversions.mapAsJavaMap(
- properties.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION));
-
- Map<String, String> offsets = JavaConversions.mapAsJavaMap(
- properties.getIndexedProperty(SPECIFIC_OFFSETS, OFFSET));
- if (partitions.isEmpty() || offsets.isEmpty()) {
- throw new ValidationException("Offsets must be set for SPECIFIC_OFFSETS mode.");
- }
- for (int i = 0; i < partitions.size(); ++i) {
- properties.validateInt(
- SPECIFIC_OFFSETS + "." + i + "." + PARTITION,
- false,
- 0,
- Integer.MAX_VALUE);
- properties.validateLong(
- SPECIFIC_OFFSETS + "." + i + "." + OFFSET,
- false,
- 0,
- Long.MAX_VALUE);
- }
- return BoxedUnit.UNIT;
- }
- };
- Map<String, Function0<BoxedUnit>> startupModeValidatorMap = new HashMap<>();
- startupModeValidatorMap.put(STARTUP_MODE_VALUE_GROUP_OFFSETS, emptyValidator);
- startupModeValidatorMap.put(STARTUP_MODE_VALUE_EARLIEST, emptyValidator);
- startupModeValidatorMap.put(STARTUP_MODE_VALUE_LATEST, emptyValidator);
- startupModeValidatorMap.put(STARTUP_MODE_VALUE_SPECIFIC_OFFSETS, specificOffsetsValidator);
-
- properties.validateEnum(STARTUP_MODE, true, toScalaImmutableMap(startupModeValidatorMap));
- validateTableJsonMapping(properties);
+ 0,
+ Integer.MAX_VALUE));
+ specificOffsetValidators.put(
+ CONNECTOR_SPECIFIC_OFFSETS_OFFSET,
+ (prefix) -> properties.validateLong(
+ prefix + CONNECTOR_SPECIFIC_OFFSETS_OFFSET,
+ false,
+ 0,
+ Long.MAX_VALUE));
+
+ final Map<String, Consumer<String>> startupModeValidation = new HashMap<>();
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS, properties.noValidation());
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_EARLIEST, properties.noValidation());
+ startupModeValidation.put(CONNECTOR_STARTUP_MODE_VALUE_LATEST, properties.noValidation());
+ startupModeValidation.put(
+ CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS,
+ prefix -> properties.validateFixedIndexedProperties(CONNECTOR_SPECIFIC_OFFSETS, false, specificOffsetValidators));
+ properties.validateEnum(CONNECTOR_STARTUP_MODE, true, startupModeValidation);
+
+ final Map<String, Consumer<String>> propertyValidators = new HashMap<>();
+ propertyValidators.put(
+ CONNECTOR_PROPERTIES_KEY,
+ prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_KEY, false, 1, Integer.MAX_VALUE));
+ propertyValidators.put(
+ CONNECTOR_PROPERTIES_VALUE,
+ prefix -> properties.validateString(prefix + CONNECTOR_PROPERTIES_VALUE, false, 0, Integer.MAX_VALUE));
+ properties.validateFixedIndexedProperties(CONNECTOR_PROPERTIES, true, propertyValidators);
}
- private void validateTableJsonMapping(DescriptorProperties properties) {
- Map<String, String> mappingTableField = JavaConversions.mapAsJavaMap(
- properties.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD));
- Map<String, String> mappingJsonField = JavaConversions.mapAsJavaMap(
- properties.getIndexedProperty(TABLE_JSON_MAPPING, JSON_FIELD));
-
- if (mappingJsonField.size() != mappingJsonField.size()) {
- throw new ValidationException("Table JSON mapping must be one to one.");
- }
-
- for (int i = 0; i < mappingTableField.size(); i++) {
- properties.validateString(
- TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD,
- false,
- 1,
- Integer.MAX_VALUE);
- properties.validateString(
- TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD,
- false,
- 1,
- Integer.MAX_VALUE);
- }
- }
+ // utilities
- @SuppressWarnings("unchecked")
- private <K, V> scala.collection.immutable.Map<K, V> toScalaImmutableMap(Map<K, V> javaMap) {
- final java.util.List<scala.Tuple2<K, V>> list = new java.util.ArrayList<>(javaMap.size());
- for (final java.util.Map.Entry<K, V> entry : javaMap.entrySet()) {
- list.add(scala.Tuple2.apply(entry.getKey(), entry.getValue()));
+ public static String normalizeStartupMode(StartupMode startupMode) {
+ switch (startupMode) {
+ case EARLIEST:
+ return CONNECTOR_STARTUP_MODE_VALUE_EARLIEST;
+ case LATEST:
+ return CONNECTOR_STARTUP_MODE_VALUE_LATEST;
+ case GROUP_OFFSETS:
+ return CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS;
+ case SPECIFIC_OFFSETS:
+ return CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS;
}
- final scala.collection.Seq<Tuple2<K, V>> seq =
- scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toSeq();
- return (scala.collection.immutable.Map<K, V>) scala.collection.immutable.Map$.MODULE$.apply(seq);
+ throw new IllegalArgumentException("Invalid startup mode.");
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
deleted file mode 100644
index 964a624..0000000
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.flink.table.api.java.StreamTableEnvironment;
-import org.apache.flink.table.descriptors.Kafka;
-
-import org.mockito.Mockito;
-
-/**
- * Tests for {@link KafkaJsonTableSourceFactory}.
- */
-public abstract class KafkaJsonTableFromDescriptorTestBase {
- private static final String GROUP_ID = "test-group";
- private static final String BOOTSTRAP_SERVERS = "localhost:1234";
- private static final String TOPIC = "test-topic";
-
- protected abstract String versionForTest();
-
- protected abstract KafkaJsonTableSource.Builder builderForTest();
-
- protected abstract void extraSettings(KafkaTableSource.Builder builder, Kafka kafka);
-
- private static StreamExecutionEnvironment env = Mockito.mock(StreamExecutionEnvironment.class);
- private static StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
-
-// @Test
-// public void buildJsonTableSourceTest() throws Exception {
-// final URL url = getClass().getClassLoader().getResource("kafka-json-schema.json");
-// Objects.requireNonNull(url);
-// final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
-//
-// Map<String, String> tableJsonMapping = new HashMap<>();
-// tableJsonMapping.put("fruit-name", "name");
-// tableJsonMapping.put("fruit-count", "count");
-// tableJsonMapping.put("event-time", "time");
-//
-// // Construct with the builder.
-// Properties props = new Properties();
-// props.put("group.id", GROUP_ID);
-// props.put("bootstrap.servers", BOOTSTRAP_SERVERS);
-//
-// Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
-// specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L);
-// specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L);
-//
-// KafkaTableSource.Builder builder = builderForTest()
-// .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(schema)))
-// .failOnMissingField(true)
-// .withTableToJsonMapping(tableJsonMapping)
-// .withKafkaProperties(props)
-// .forTopic(TOPIC)
-// .fromSpecificOffsets(specificOffsets)
-// .withSchema(
-// TableSchema.builder()
-// .field("fruit-name", Types.STRING)
-// .field("fruit-count", Types.INT)
-// .field("event-time", Types.LONG)
-// .field("proc-time", Types.SQL_TIMESTAMP)
-// .build())
-// .withProctimeAttribute("proc-time");
-//
-// // Construct with the descriptor.
-// Map<Integer, Long> offsets = new HashMap<>();
-// offsets.put(0, 100L);
-// offsets.put(1, 123L);
-// Kafka kafka = new Kafka()
-// .version(versionForTest())
-// .groupId(GROUP_ID)
-// .bootstrapServers(BOOTSTRAP_SERVERS)
-// .topic(TOPIC)
-// .startupMode(StartupMode.SPECIFIC_OFFSETS)
-// .specificOffsets(offsets)
-// .tableJsonMapping(tableJsonMapping);
-// extraSettings(builder, kafka);
-//
-// TableSource source = tEnv
-// .from(kafka)
-// .withFormat(
-// new Json()
-// .schema(schema)
-// .failOnMissingField(true))
-// .withSchema(new Schema()
-// .field("fruit-name", Types.STRING)
-// .field("fruit-count", Types.INT)
-// .field("event-time", Types.LONG)
-// .field("proc-time", Types.SQL_TIMESTAMP).proctime())
-// .toTableSource();
-//
-// Assert.assertEquals(builder.build(), source);
-// }
-
-// @Test(expected = TableException.class)
-// public void buildJsonTableSourceFailTest() {
-// tEnv.from(
-// new Kafka()
-// .version(versionForTest())
-// .groupId(GROUP_ID)
-// .bootstrapServers(BOOTSTRAP_SERVERS)
-// .topic(TOPIC)
-// .startupMode(StartupMode.SPECIFIC_OFFSETS)
-// .specificOffsets(new HashMap<>()))
-// .withFormat(
-// new Json()
-// .schema("")
-// .failOnMissingField(true))
-// .toTableSource();
-// }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
new file mode 100644
index 0000000..2b081a9
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactoryTestBase.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.formats.json.JsonSchemaConverter;
+import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.descriptors.FormatDescriptor;
+import org.apache.flink.table.descriptors.Json;
+import org.apache.flink.table.descriptors.Kafka;
+import org.apache.flink.table.descriptors.Schema;
+import org.apache.flink.table.descriptors.TestTableSourceDescriptor;
+import org.apache.flink.table.sources.TableSource;
+import org.apache.flink.table.sources.TableSourceFactoryService;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link KafkaJsonTableSourceFactory}.
+ */
+public abstract class KafkaJsonTableSourceFactoryTestBase {
+
+ private static final String JSON_SCHEMA =
+ "{" +
+ " 'title': 'Fruit'," +
+ " 'type': 'object'," +
+ " 'properties': {" +
+ " 'name': {" +
+ " 'type': 'string'" +
+ " }," +
+ " 'count': {" +
+ " 'type': 'integer'" +
+ " }," +
+ " 'time': {" +
+ " 'description': 'Age in years'," +
+ " 'type': 'number'" +
+ " }" + " }," +
+ " 'required': ['name', 'count', 'time']" +
+ "}";
+
+ private static final String TOPIC = "test-topic";
+
+ protected abstract String version();
+
+ protected abstract KafkaJsonTableSource.Builder builder();
+
+ @Test
+ public void testTableSourceFromJsonSchema() {
+ testTableSource(
+ new Json()
+ .jsonSchema(JSON_SCHEMA)
+ .failOnMissingField(true)
+ );
+ }
+
+ @Test
+ public void testTableSourceDerivedSchema() {
+ testTableSource(
+ new Json()
+ .deriveSchema()
+ .failOnMissingField(true)
+ );
+ }
+
+ private void testTableSource(FormatDescriptor format) {
+ // construct table source using a builder
+
+ final Map<String, String> tableJsonMapping = new HashMap<>();
+ tableJsonMapping.put("fruit-name", "name");
+ tableJsonMapping.put("count", "count");
+ tableJsonMapping.put("event-time", "time");
+
+ final Properties props = new Properties();
+ props.put("group.id", "test-group");
+ props.put("bootstrap.servers", "localhost:1234");
+
+ final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
+ specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L);
+ specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L);
+
+ final KafkaTableSource builderSource = builder()
+ .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(JSON_SCHEMA)))
+ .failOnMissingField(true)
+ .withTableToJsonMapping(tableJsonMapping)
+ .withKafkaProperties(props)
+ .forTopic(TOPIC)
+ .fromSpecificOffsets(specificOffsets)
+ .withSchema(
+ TableSchema.builder()
+ .field("fruit-name", Types.STRING)
+ .field("count", Types.BIG_INT)
+ .field("event-time", Types.BIG_DEC)
+ .field("proc-time", Types.SQL_TIMESTAMP)
+ .build())
+ .withProctimeAttribute("proc-time")
+ .build();
+
+ // construct table source using descriptors and table source factory
+
+ final Map<Integer, Long> offsets = new HashMap<>();
+ offsets.put(0, 100L);
+ offsets.put(1, 123L);
+
+ final TestTableSourceDescriptor testDesc = new TestTableSourceDescriptor(
+ new Kafka()
+ .version(version())
+ .topic(TOPIC)
+ .properties(props)
+ .startFromSpecificOffsets(offsets))
+ .addFormat(format)
+ .addSchema(
+ new Schema()
+ .field("fruit-name", Types.STRING).from("name")
+ .field("count", Types.BIG_INT) // no from so it must match with the input
+ .field("event-time", Types.BIG_DEC).from("time")
+ .field("proc-time", Types.SQL_TIMESTAMP).proctime());
+
+ final TableSource<?> factorySource = TableSourceFactoryService.findAndCreateTableSource(testDesc);
+
+ assertEquals(builderSource, factorySource);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java
new file mode 100644
index 0000000..f3d96f1
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/table/descriptors/KafkaTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Tests for the {@link Kafka} descriptor.
+ */
+public class KafkaTest extends DescriptorTestBase {
+
+ @Override
+ public List<Descriptor> descriptors() {
+ final Descriptor earliestDesc =
+ new Kafka()
+ .version("0.8")
+ .startFromEarliest()
+ .topic("WhateverTopic");
+
+ final Descriptor specificOffsetsDesc =
+ new Kafka()
+ .version("0.11")
+ .topic("MyTable")
+ .startFromSpecificOffset(0, 42L)
+ .startFromSpecificOffset(1, 300L)
+ .property("zookeeper.stuff", "12")
+ .property("kafka.stuff", "42");
+
+ final Map<Integer, Long> offsets = new HashMap<>();
+ offsets.put(0, 42L);
+ offsets.put(1, 300L);
+
+ final Properties properties = new Properties();
+ properties.put("zookeeper.stuff", "12");
+ properties.put("kafka.stuff", "42");
+
+ final Descriptor specificOffsetsMapDesc =
+ new Kafka()
+ .version("0.11")
+ .topic("MyTable")
+ .startFromSpecificOffsets(offsets)
+ .properties(properties);
+
+ return Arrays.asList(earliestDesc, specificOffsetsDesc, specificOffsetsMapDesc);
+ }
+
+ @Override
+ public List<Map<String, String>> properties() {
+ final Map<String, String> props1 = new HashMap<>();
+ props1.put("connector.property-version", "1");
+ props1.put("connector.type", "kafka");
+ props1.put("connector.version", "0.8");
+ props1.put("connector.topic", "WhateverTopic");
+ props1.put("connector.startup-mode", "earliest-offset");
+
+ final Map<String, String> props2 = new HashMap<>();
+ props2.put("connector.property-version", "1");
+ props2.put("connector.type", "kafka");
+ props2.put("connector.version", "0.11");
+ props2.put("connector.topic", "MyTable");
+ props2.put("connector.startup-mode", "specific-offsets");
+ props2.put("connector.specific-offsets.0.partition", "0");
+ props2.put("connector.specific-offsets.0.offset", "42");
+ props2.put("connector.specific-offsets.1.partition", "1");
+ props2.put("connector.specific-offsets.1.offset", "300");
+ props2.put("connector.properties.0.key", "zookeeper.stuff");
+ props2.put("connector.properties.0.value", "12");
+ props2.put("connector.properties.1.key", "kafka.stuff");
+ props2.put("connector.properties.1.value", "42");
+
+ final Map<String, String> props3 = new HashMap<>();
+ props3.put("connector.property-version", "1");
+ props3.put("connector.type", "kafka");
+ props3.put("connector.version", "0.11");
+ props3.put("connector.topic", "MyTable");
+ props3.put("connector.startup-mode", "specific-offsets");
+ props3.put("connector.specific-offsets.0.partition", "0");
+ props3.put("connector.specific-offsets.0.offset", "42");
+ props3.put("connector.specific-offsets.1.partition", "1");
+ props3.put("connector.specific-offsets.1.offset", "300");
+ props3.put("connector.properties.0.key", "zookeeper.stuff");
+ props3.put("connector.properties.0.value", "12");
+ props3.put("connector.properties.1.key", "kafka.stuff");
+ props3.put("connector.properties.1.value", "42");
+
+ return Arrays.asList(props1, props2, props3);
+ }
+
+ @Override
+ public DescriptorValidator validator() {
+ return new KafkaValidator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
deleted file mode 100644
index 5167e5e..0000000
--- a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-{
- "title": "Fruit",
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "count": {
- "type": "integer"
- },
- "time": {
- "description": "Age in years",
- "type": "number"
- }
- },
- "required": ["name", "count", "time"]
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index c555f66..90e79ea 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -337,22 +337,6 @@ under the License.
</dependency>
<!-- end optional Flink libraries -->
- <!-- start optional Flink formats -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-avro</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <!-- end optional Flink formats -->
-
<!-- test dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-formats/flink-json/pom.xml
----------------------------------------------------------------------
diff --git a/flink-formats/flink-json/pom.xml b/flink-formats/flink-json/pom.xml
index ccc48f4..1c77a45 100644
--- a/flink-formats/flink-json/pom.xml
+++ b/flink-formats/flink-json/pom.xml
@@ -50,6 +50,33 @@ under the License.
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <!-- use a dedicated Scala version to not depend on it -->
+ <artifactId>flink-table_2.11</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <!-- Projects depending on this project, won't depend on flink-table. -->
+ <optional>true</optional>
+ </dependency>
+
+ <!-- test dependencies -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <!-- use a dedicated Scala version to not depend on it -->
+ <artifactId>flink-table_2.11</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- flink-table needs Scala -->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
----------------------------------------------------------------------
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
new file mode 100644
index 0000000..9c12191
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/Json.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.typeutils.TypeStringUtils;
+import org.apache.flink.util.Preconditions;
+
+import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_JSON_SCHEMA;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE;
+
+/**
+ * Format descriptor for JSON.
+ */
+public class Json extends FormatDescriptor {
+
+ private Boolean failOnMissingField;
+ private Boolean deriveSchema;
+ private String jsonSchema;
+ private String schema;
+
+ /**
+ * Format descriptor for JSON.
+ */
+ public Json() {
+ super(FORMAT_TYPE_VALUE, 1);
+ }
+
+ /**
+ * Sets flag whether to fail if a field is missing or not.
+ *
+ * @param failOnMissingField If set to true, the operation fails if there is a missing field.
+ * If set to false, a missing field is set to null.
+ */
+ public Json failOnMissingField(boolean failOnMissingField) {
+ this.failOnMissingField = failOnMissingField;
+ return this;
+ }
+
+ /**
+ * Sets the JSON schema string with field names and the types according to the JSON schema
+ * specification [[http://json-schema.org/specification.html]].
+ *
+ * <p>The schema might be nested.
+ *
+ * @param jsonSchema JSON schema
+ */
+ public Json jsonSchema(String jsonSchema) {
+ Preconditions.checkNotNull(jsonSchema);
+ this.jsonSchema = jsonSchema;
+ this.schema = null;
+ this.deriveSchema = null;
+ return this;
+ }
+
+ /**
+ * Sets the schema using type information.
+ *
+ * <p>JSON objects are represented as ROW types.
+ *
+ * <p>The schema might be nested.
+ *
+ * @param schemaType type information that describes the schema
+ */
+ public Json schema(TypeInformation<?> schemaType) {
+ Preconditions.checkNotNull(schemaType);
+ this.schema = TypeStringUtils.writeTypeInfo(schemaType);
+ this.jsonSchema = null;
+ this.deriveSchema = null;
+ return this;
+ }
+
+ /**
+ * Derives the format schema from the table's schema described using {@link Schema}.
+ *
+ * <p>This allows for defining schema information only once.
+ *
+ * <p>The names, types, and field order of the format are determined by the table's
+ * schema. Time attributes are ignored. A "from" definition is interpreted as a field renaming
+ * in the format.
+ */
+ public Json deriveSchema() {
+ this.deriveSchema = true;
+ this.schema = null;
+ this.jsonSchema = null;
+ return this;
+ }
+
+ /**
+ * Internal method for format properties conversion.
+ */
+ @Override
+ public void addFormatProperties(DescriptorProperties properties) {
+ if (deriveSchema != null) {
+ properties.putBoolean(FORMAT_DERIVE_SCHEMA(), deriveSchema);
+ }
+
+ if (jsonSchema != null) {
+ properties.putString(FORMAT_JSON_SCHEMA, jsonSchema);
+ }
+
+ if (schema != null) {
+ properties.putString(FORMAT_SCHEMA, schema);
+ }
+
+ if (failOnMissingField != null) {
+ properties.putBoolean(FORMAT_FAIL_ON_MISSING_FIELD, failOnMissingField);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
----------------------------------------------------------------------
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
new file mode 100644
index 0000000..fea7cf5
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/table/descriptors/JsonValidator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.table.api.ValidationException;
+
+/**
+ * Validator for {@link Json}.
+ */
+public class JsonValidator extends FormatDescriptorValidator {
+
+ public static final String FORMAT_TYPE_VALUE = "json";
+ public static final String FORMAT_SCHEMA = "format.schema";
+ public static final String FORMAT_JSON_SCHEMA = "format.json-schema";
+ public static final String FORMAT_FAIL_ON_MISSING_FIELD = "format.fail-on-missing-field";
+
+ @Override
+ public void validate(DescriptorProperties properties) {
+ super.validate(properties);
+ properties.validateBoolean(FORMAT_DERIVE_SCHEMA(), true);
+ final boolean deriveSchema = properties.getOptionalBoolean(FORMAT_DERIVE_SCHEMA()).orElse(false);
+ final boolean hasSchema = properties.containsKey(FORMAT_SCHEMA);
+ final boolean hasSchemaString = properties.containsKey(FORMAT_JSON_SCHEMA);
+ if (deriveSchema && (hasSchema || hasSchemaString)) {
+ throw new ValidationException(
+ "Format cannot define a schema and derive from the table's schema at the same time.");
+ } else if (!deriveSchema && hasSchema && hasSchemaString) {
+ throw new ValidationException("A definition of both a schema and JSON schema is not allowed.");
+ } else if (!deriveSchema && !hasSchema && !hasSchemaString) {
+ throw new ValidationException("A definition of a schema or JSON schema is required.");
+ } else if (hasSchema) {
+ properties.validateType(FORMAT_SCHEMA, false);
+ } else if (hasSchemaString) {
+ properties.validateString(FORMAT_JSON_SCHEMA, false, 1);
+ }
+
+ properties.validateBoolean(FORMAT_FAIL_ON_MISSING_FIELD, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java
----------------------------------------------------------------------
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java
new file mode 100644
index 0000000..6e370a0
--- /dev/null
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/table/descriptors/JsonTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.api.Types;
+import org.apache.flink.table.api.ValidationException;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests for the {@link Json} descriptor.
+ */
+public class JsonTest extends DescriptorTestBase {
+
+ private static final String JSON_SCHEMA =
+ "{" +
+ " 'title': 'Person'," +
+ " 'type': 'object'," +
+ " 'properties': {" +
+ " 'firstName': {" +
+ " 'type': 'string'" +
+ " }," +
+ " 'lastName': {" +
+ " 'type': 'string'" +
+ " }," +
+ " 'age': {" +
+ " 'description': 'Age in years'," +
+ " 'type': 'integer'," +
+ " 'minimum': 0" +
+ " }" +
+ " }," +
+ " 'required': ['firstName', 'lastName']" +
+ "}";
+
+ @Test(expected = ValidationException.class)
+ public void testInvalidMissingField() {
+ addPropertyAndVerify(descriptors().get(0), "format.fail-on-missing-field", "DDD");
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testMissingSchema() {
+ removePropertyAndVerify(descriptors().get(0), "format.json-schema");
+ }
+
+ @Test(expected = ValidationException.class)
+ public void testDuplicateSchema() {
+ // we add an additional non-json schema
+ addPropertyAndVerify(descriptors().get(0), "format.schema", "DDD");
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public List<Descriptor> descriptors() {
+ final Descriptor desc1 = new Json().jsonSchema("test");
+
+ final Descriptor desc2 = new Json().jsonSchema(JSON_SCHEMA).failOnMissingField(true);
+
+ final Descriptor desc3 = new Json()
+ .schema(
+ Types.ROW(
+ new String[]{"test1", "test2"},
+ new TypeInformation[]{Types.STRING(), Types.SQL_TIMESTAMP()}))
+ .failOnMissingField(true);
+
+ final Descriptor desc4 = new Json().deriveSchema();
+
+ return Arrays.asList(desc1, desc2, desc3, desc4);
+ }
+
+ @Override
+ public List<Map<String, String>> properties() {
+ final Map<String, String> props1 = new HashMap<>();
+ props1.put("format.type", "json");
+ props1.put("format.property-version", "1");
+ props1.put("format.json-schema", "test");
+
+ final Map<String, String> props2 = new HashMap<>();
+ props2.put("format.type", "json");
+ props2.put("format.property-version", "1");
+ props2.put("format.json-schema", JSON_SCHEMA);
+ props2.put("format.fail-on-missing-field", "true");
+
+ final Map<String, String> props3 = new HashMap<>();
+ props3.put("format.type", "json");
+ props3.put("format.property-version", "1");
+ props3.put("format.schema", "ROW(test1 VARCHAR, test2 TIMESTAMP)");
+ props3.put("format.fail-on-missing-field", "true");
+
+ final Map<String, String> props4 = new HashMap<>();
+ props4.put("format.type", "json");
+ props4.put("format.property-version", "1");
+ props4.put("format.derive-schema", "true");
+
+ return Arrays.asList(props1, props2, props3, props4);
+ }
+
+ @Override
+ public DescriptorValidator validator() {
+ return new JsonValidator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
index 9e7413c..8c40885 100644
--- a/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
+++ b/flink-libraries/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java
@@ -476,7 +476,7 @@ public class LocalExecutor implements Executor {
}
env.getSources().forEach((name, source) -> {
- TableSource<?> tableSource = TableSourceFactoryService.findTableSourceFactory(source);
+ TableSource<?> tableSource = TableSourceFactoryService.findAndCreateTableSource(source);
tableEnv.registerTableSource(name, tableSource);
});
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/pom.xml
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/pom.xml b/flink-libraries/flink-table/pom.xml
index bf6585a..2c7888e 100644
--- a/flink-libraries/flink-table/pom.xml
+++ b/flink-libraries/flink-table/pom.xml
@@ -317,6 +317,18 @@ under the License.
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
<!-- Scala Code Style, most of the configuration done via plugin management -->
<plugin>
<groupId>org.scalastyle</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties b/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties
index 86a48a8..ec46579 100644
--- a/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties
+++ b/flink-libraries/flink-table/src/main/resources/tableSourceConverter.properties
@@ -17,6 +17,13 @@
################################################################################
################################################################################
+# NOTE: THIS APPROACH IS DEPRECATED AND WILL BE REMOVED IN FUTURE VERSIONS!
+#
+# We recommend to use a org.apache.flink.table.sources.TableSourceFactory
+# instead. They allow to define new factories by using Java Service Providers.
+################################################################################
+
+################################################################################
# The config file is used to specify the packages of current module where
# to find TableSourceConverter implementation class annotated with TableType.
# If there are multiple packages to scan, put those packages together into a
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
index 1e88d93..6958b3d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
@@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeutils.CompositeType
import _root_.scala.collection.mutable.ArrayBuffer
+import _root_.java.util.Objects
/**
* A TableSchema represents a Table's structure.
@@ -94,7 +95,7 @@ class TableSchema(
/**
* Returns the number of columns.
*/
- def getColumnNum: Int = columnNames.length
+ def getColumnCount: Int = columnNames.length
/**
* Returns all column names as an array.
@@ -134,6 +135,9 @@ class TableSchema(
def canEqual(other: Any): Boolean = other.isInstanceOf[TableSchema]
+ override def hashCode(): Int = {
+ Objects.hash(columnNames, columnTypes)
+ }
}
object TableSchema {
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
index fc7f7a3..ef14b8a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalCatalogTable.scala
@@ -23,6 +23,7 @@ import java.util.{HashMap => JHashMap, Map => JMap}
import org.apache.flink.table.api.{TableException, TableSchema}
import org.apache.flink.table.catalog.ExternalCatalogTable._
+import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME}
import org.apache.flink.table.descriptors._
import org.apache.flink.table.plan.stats.TableStats
@@ -73,8 +74,7 @@ class ExternalCatalogTable(
lazy val tableType: String = {
val props = new DescriptorProperties()
connectorDesc.addProperties(props)
- props
- .getString(CONNECTOR_LEGACY_TYPE)
+ toScala(props.getOptionalString(CONNECTOR_LEGACY_TYPE))
.getOrElse(throw new TableException("Could not find a legacy table type to return."))
}
@@ -88,8 +88,7 @@ class ExternalCatalogTable(
lazy val schema: TableSchema = {
val props = new DescriptorProperties()
connectorDesc.addProperties(props)
- props
- .getTableSchema(CONNECTOR_LEGACY_SCHEMA)
+ toScala(props.getOptionalTableSchema(CONNECTOR_LEGACY_SCHEMA))
.getOrElse(throw new TableException("Could not find a legacy schema to return."))
}
@@ -105,7 +104,7 @@ class ExternalCatalogTable(
val props = new DescriptorProperties(normalizeKeys = false)
val legacyProps = new JHashMap[String, String]()
connectorDesc.addProperties(props)
- props.asMap.flatMap { case (k, v) =>
+ props.asMap.asScala.flatMap { case (k, v) =>
if (k.startsWith(CONNECTOR_LEGACY_PROPERTY)) {
// remove "connector.legacy-property-"
Some(legacyProps.put(k.substring(CONNECTOR_LEGACY_PROPERTY.length + 1), v))
@@ -138,7 +137,7 @@ class ExternalCatalogTable(
metadataDesc match {
case Some(meta) =>
meta.addProperties(normalizedProps)
- normalizedProps.getString(METADATA_COMMENT).orNull
+ normalizedProps.getOptionalString(METADATA_COMMENT).orElse(null)
case None =>
null
}
@@ -157,7 +156,7 @@ class ExternalCatalogTable(
metadataDesc match {
case Some(meta) =>
meta.addProperties(normalizedProps)
- normalizedProps.getLong(METADATA_CREATION_TIME).map(v => Long.box(v)).orNull
+ normalizedProps.getOptionalLong(METADATA_CREATION_TIME).orElse(null)
case None =>
null
}
@@ -176,7 +175,7 @@ class ExternalCatalogTable(
metadataDesc match {
case Some(meta) =>
meta.addProperties(normalizedProps)
- normalizedProps.getLong(METADATA_LAST_ACCESS_TIME).map(v => Long.box(v)).orNull
+ normalizedProps.getOptionalLong(METADATA_LAST_ACCESS_TIME).orElse(null)
case None =>
null
}
@@ -267,7 +266,7 @@ object ExternalCatalogTable {
tableType: String,
schema: TableSchema,
legacyProperties: JMap[String, String])
- extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1) {
+ extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1, formatNeeded = false) {
override protected def addConnectorProperties(properties: DescriptorProperties): Unit = {
properties.putString(CONNECTOR_LEGACY_TYPE, tableType)
@@ -276,8 +275,6 @@ object ExternalCatalogTable {
properties.putString(s"$CONNECTOR_LEGACY_PROPERTY-$k", v)
}
}
-
- override private[flink] def needsFormat() = false
}
def toConnectorDescriptor(
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
index 3bc5dc0..2288522 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/catalog/ExternalTableSourceUtil.scala
@@ -58,7 +58,7 @@ object ExternalTableSourceUtil extends Logging {
}
// use the factory approach
else {
- val source = TableSourceFactoryService.findTableSourceFactory(externalCatalogTable)
+ val source = TableSourceFactoryService.findAndCreateTableSource(externalCatalogTable)
tableEnv match {
// check for a batch table source in this batch environment
case _: BatchTableEnvironment =>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala
index ed9ee7d..afdd84c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/BatchTableSourceDescriptor.scala
@@ -43,7 +43,7 @@ class BatchTableSourceDescriptor(tableEnv: BatchTableEnvironment, connector: Con
* Searches for the specified table source, configures it accordingly, and returns it.
*/
def toTableSource: TableSource[_] = {
- val source = TableSourceFactoryService.findTableSourceFactory(this)
+ val source = TableSourceFactoryService.findAndCreateTableSource(this)
source match {
case _: BatchTableSource[_] => source
case _ => throw new TableException(
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
index f691b4f..dc344f3 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptor.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION}
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
/**
* Describes a connector to an other system.
@@ -27,7 +27,8 @@ import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTO
*/
abstract class ConnectorDescriptor(
private val tpe: String,
- private val version: Int)
+ private val version: Int,
+ private val formatNeeded: Boolean)
extends Descriptor {
override def toString: String = this.getClass.getSimpleName
@@ -37,7 +38,7 @@ abstract class ConnectorDescriptor(
*/
final private[flink] def addProperties(properties: DescriptorProperties): Unit = {
properties.putString(CONNECTOR_TYPE, tpe)
- properties.putLong(CONNECTOR_VERSION, version)
+ properties.putLong(CONNECTOR_PROPERTY_VERSION, version)
addConnectorProperties(properties)
}
@@ -49,6 +50,6 @@ abstract class ConnectorDescriptor(
/**
* Internal method that defines if this connector requires a format descriptor.
*/
- private[flink] def needsFormat(): Boolean
+ private[flink] def needsFormat(): Boolean = formatNeeded
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
index 8ab0f45..211d374 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/ConnectorDescriptorValidator.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION}
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_PROPERTY_VERSION}
/**
* Validator for [[ConnectorDescriptor]].
@@ -27,13 +27,27 @@ class ConnectorDescriptorValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
properties.validateString(CONNECTOR_TYPE, isOptional = false, minLen = 1)
- properties.validateInt(CONNECTOR_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateInt(CONNECTOR_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
}
}
object ConnectorDescriptorValidator {
+ /**
+ * Key for describing the type of the connector. Usually used for factory discovery.
+ */
val CONNECTOR_TYPE = "connector.type"
+
+ /**
+ * Key for describing the property version. This property can be used for backwards
+ * compatibility in case the property format changes.
+ */
+ val CONNECTOR_PROPERTY_VERSION = "connector.property-version"
+
+ /**
+ * Key for describing the version of the connector. This property can be used for different
+ * connector versions (e.g. Kafka 0.8 or Kafka 0.11).
+ */
val CONNECTOR_VERSION = "connector.version"
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
index 0493d99..7e69fea 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Csv.scala
@@ -23,6 +23,7 @@ import org.apache.flink.table.api.{TableSchema, ValidationException}
import org.apache.flink.table.descriptors.CsvValidator._
import scala.collection.mutable
+import scala.collection.JavaConverters._
/**
* Format descriptor for comma-separated values (CSV).
@@ -31,7 +32,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
private var fieldDelim: Option[String] = None
private var lineDelim: Option[String] = None
- private val formatSchema: mutable.LinkedHashMap[String, String] =
+ private val schema: mutable.LinkedHashMap[String, String] =
mutable.LinkedHashMap[String, String]()
private var quoteCharacter: Option[Character] = None
private var commentPrefix: Option[String] = None
@@ -67,7 +68,7 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
* @param schema the table schema
*/
def schema(schema: TableSchema): Csv = {
- this.formatSchema.clear()
+ this.schema.clear()
DescriptorProperties.normalizeTableSchema(schema).foreach {
case (n, t) => field(n, t)
}
@@ -96,10 +97,10 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
* @param fieldType the type string of the field
*/
def field(fieldName: String, fieldType: String): Csv = {
- if (formatSchema.contains(fieldName)) {
+ if (schema.contains(fieldName)) {
throw new ValidationException(s"Duplicate field name $fieldName.")
}
- formatSchema += (fieldName -> fieldType)
+ schema += (fieldName -> fieldType)
this
}
@@ -145,7 +146,9 @@ class Csv extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
override protected def addFormatProperties(properties: DescriptorProperties): Unit = {
fieldDelim.foreach(properties.putString(FORMAT_FIELD_DELIMITER, _))
lineDelim.foreach(properties.putString(FORMAT_LINE_DELIMITER, _))
- properties.putTableSchema(FORMAT_FIELDS, formatSchema.toIndexedSeq)
+ properties.putTableSchema(
+ FORMAT_FIELDS,
+ schema.toIndexedSeq.map(DescriptorProperties.toJava[String, String]).asJava)
quoteCharacter.foreach(properties.putCharacter(FORMAT_QUOTE_CHARACTER, _))
commentPrefix.foreach(properties.putString(FORMAT_COMMENT_PREFIX, _))
isIgnoreFirstLine.foreach(properties.putBoolean(FORMAT_IGNORE_FIRST_LINE, _))
[4/5] flink git commit: [FLINK-8538] [table] Improve unified table
sources
Posted by tw...@apache.org.
[FLINK-8538] [table] Improve unified table sources
This closes #5564.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/db2c510f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/db2c510f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/db2c510f
Branch: refs/heads/release-1.5
Commit: db2c510fb4f171c9e9940759e5fbaf466ec74474
Parents: 1d26062
Author: Timo Walther <tw...@apache.org>
Authored: Mon Feb 19 13:35:45 2018 +0100
Committer: Timo Walther <tw...@apache.org>
Committed: Tue Feb 27 20:23:00 2018 +0100
----------------------------------------------------------------------
.../flink-connector-kafka-0.10/pom.xml | 8 +
.../kafka/Kafka010JsonTableSourceFactory.java | 5 +-
.../resources/tableSourceConverter.properties | 29 -
.../Kafka010JsonTableSourceFactoryTest.java | 37 +
.../kafka/Kafka010TableSourceFactoryTest.java | 41 -
.../flink-connector-kafka-0.11/pom.xml | 8 +
.../kafka/Kafka011JsonTableSourceFactory.java | 5 +-
.../resources/tableSourceConverter.properties | 29 -
.../Kafka011JsonTableSourceFactoryTest.java | 37 +
.../kafka/Kafka011TableSourceFactoryTest.java | 41 -
.../flink-connector-kafka-0.8/pom.xml | 8 +
.../kafka/Kafka08JsonTableSourceFactory.java | 5 +-
.../resources/tableSourceConverter.properties | 29 -
.../Kafka08JsonTableSourceFactoryTest.java | 37 +
.../kafka/Kafka08TableSourceFactoryTest.java | 42 -
.../flink-connector-kafka-0.9/pom.xml | 8 +
.../kafka/Kafka09JsonTableSourceFactory.java | 5 +-
.../resources/tableSourceConverter.properties | 29 -
.../Kafka09JsonTableSourceFactoryTest.java | 37 +
.../kafka/Kafka09TableSourceFactoryTest.java | 41 -
.../flink-connector-kafka-base/pom.xml | 16 +-
.../connectors/kafka/KafkaJsonTableSource.java | 24 +-
.../kafka/KafkaJsonTableSourceFactory.java | 267 +++---
.../connectors/kafka/KafkaTableSource.java | 42 +-
.../apache/flink/table/descriptors/Kafka.java | 239 +++---
.../flink/table/descriptors/KafkaValidator.java | 226 ++---
.../KafkaJsonTableFromDescriptorTestBase.java | 127 ---
.../KafkaJsonTableSourceFactoryTestBase.java | 145 ++++
.../flink/table/descriptors/KafkaTest.java | 113 +++
.../src/test/resources/kafka-json-schema.json | 35 -
flink-dist/pom.xml | 16 -
flink-formats/flink-json/pom.xml | 29 +-
.../apache/flink/table/descriptors/Json.java | 129 +++
.../flink/table/descriptors/JsonValidator.java | 55 ++
.../flink/table/descriptors/JsonTest.java | 124 +++
.../client/gateway/local/LocalExecutor.java | 2 +-
flink-libraries/flink-table/pom.xml | 12 +
.../resources/tableSourceConverter.properties | 7 +
.../apache/flink/table/api/TableSchema.scala | 6 +-
.../table/catalog/ExternalCatalogTable.scala | 19 +-
.../table/catalog/ExternalTableSourceUtil.scala | 2 +-
.../BatchTableSourceDescriptor.scala | 2 +-
.../table/descriptors/ConnectorDescriptor.scala | 9 +-
.../ConnectorDescriptorValidator.scala | 18 +-
.../apache/flink/table/descriptors/Csv.scala | 13 +-
.../descriptors/DescriptorProperties.scala | 833 ++++++++++++++++---
.../flink/table/descriptors/FileSystem.scala | 5 +-
.../table/descriptors/FormatDescriptor.scala | 4 +-
.../descriptors/FormatDescriptorValidator.scala | 23 +-
.../apache/flink/table/descriptors/Json.scala | 78 --
.../flink/table/descriptors/JsonValidator.scala | 41 -
.../table/descriptors/MetadataValidator.scala | 6 +-
.../flink/table/descriptors/Rowtime.scala | 8 +-
.../table/descriptors/RowtimeValidator.scala | 179 ++--
.../apache/flink/table/descriptors/Schema.scala | 14 +-
.../table/descriptors/SchemaValidator.scala | 171 +++-
.../flink/table/descriptors/Statistics.scala | 6 +-
.../table/descriptors/StatisticsValidator.scala | 21 +-
.../StreamTableSourceDescriptor.scala | 2 +-
.../descriptors/TableSourceDescriptor.scala | 3 +-
.../table/sources/CsvTableSourceFactory.scala | 43 +-
.../table/sources/TableSourceFactory.scala | 11 +-
.../sources/TableSourceFactoryService.scala | 26 +-
.../tsextractors/StreamRecordTimestamp.scala | 5 +-
.../flink/table/descriptors/CsvTest.scala | 83 +-
.../table/descriptors/DescriptorTestBase.scala | 65 +-
.../table/descriptors/FileSystemTest.scala | 33 +-
.../flink/table/descriptors/JsonTest.scala | 77 --
.../flink/table/descriptors/MetadataTest.scala | 38 +-
.../flink/table/descriptors/RowtimeTest.scala | 59 +-
.../flink/table/descriptors/SchemaTest.scala | 84 +-
.../table/descriptors/SchemaValidatorTest.scala | 76 ++
.../table/descriptors/StatisticsTest.scala | 72 +-
.../sources/TableSourceFactoryServiceTest.scala | 20 +-
.../table/sources/TestTableSourceFactory.scala | 8 +-
75 files changed, 2681 insertions(+), 1571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.10/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/pom.xml b/flink-connectors/flink-connector-kafka-0.10/pom.xml
index 8b4ff38..2d273ae 100644
--- a/flink-connectors/flink-connector-kafka-0.10/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.10/pom.xml
@@ -192,6 +192,14 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
index 1d03f6c..c639a44 100644
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
@@ -18,12 +18,13 @@
package org.apache.flink.streaming.connectors.kafka;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_010;
/**
* Factory for creating configured instances of {@link Kafka010JsonTableSource}.
*/
public class Kafka010JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+
@Override
protected KafkaJsonTableSource.Builder createBuilder() {
return new Kafka010JsonTableSource.Builder();
@@ -31,6 +32,6 @@ public class Kafka010JsonTableSourceFactory extends KafkaJsonTableSourceFactory
@Override
protected String kafkaVersion() {
- return KAFKA_VERSION_VALUE_010;
+ return CONNECTOR_VERSION_VALUE_010;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
deleted file mode 100644
index 5409b49..0000000
--- a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-################################################################################
-# The config file is used to specify the packages of current module where
-# to find TableSourceConverter implementation class annotated with TableType.
-# If there are multiple packages to scan, put those packages together into a
-# string separated with ',', for example, org.package1,org.package2.
-# Please notice:
-# It's better to have a tableSourceConverter.properties in each connector Module
-# which offers converters instead of put all information into the
-# tableSourceConverter.properties of flink-table module.
-################################################################################
-scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java
new file mode 100644
index 0000000..22cf659
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactoryTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_010;
+
+/**
+ * Tests for {@link Kafka010JsonTableSourceFactory}.
+ */
+public class Kafka010JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase {
+
+ @Override
+ protected String version() {
+ return CONNECTOR_VERSION_VALUE_010;
+ }
+
+ @Override
+ protected KafkaJsonTableSource.Builder builder() {
+ return Kafka010JsonTableSource.builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
deleted file mode 100644
index 15b89e8..0000000
--- a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.table.descriptors.Kafka;
-
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010;
-
-/**
- * Tests for {@link Kafka010JsonTableSourceFactory}.
- */
-public class Kafka010TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
- protected String versionForTest() {
- return KAFKA_VERSION_VALUE_010;
- }
-
- protected KafkaJsonTableSource.Builder builderForTest() {
- return Kafka010JsonTableSource.builder();
- }
-
- @Override
- protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
- // no extra settings
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.11/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/pom.xml b/flink-connectors/flink-connector-kafka-0.11/pom.xml
index 1e935f6..0fc1f13 100644
--- a/flink-connectors/flink-connector-kafka-0.11/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.11/pom.xml
@@ -201,6 +201,14 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
index ca4d6ce..6745bb2 100644
--- a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
@@ -18,12 +18,13 @@
package org.apache.flink.streaming.connectors.kafka;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_011;
/**
* Factory for creating configured instances of {@link Kafka011JsonTableSource}.
*/
public class Kafka011JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+
@Override
protected KafkaJsonTableSource.Builder createBuilder() {
return new Kafka011JsonTableSource.Builder();
@@ -31,6 +32,6 @@ public class Kafka011JsonTableSourceFactory extends KafkaJsonTableSourceFactory
@Override
protected String kafkaVersion() {
- return KAFKA_VERSION_VALUE_011;
+ return CONNECTOR_VERSION_VALUE_011;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
deleted file mode 100644
index 5409b49..0000000
--- a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-################################################################################
-# The config file is used to specify the packages of current module where
-# to find TableSourceConverter implementation class annotated with TableType.
-# If there are multiple packages to scan, put those packages together into a
-# string separated with ',', for example, org.package1,org.package2.
-# Please notice:
-# It's better to have a tableSourceConverter.properties in each connector Module
-# which offers converters instead of put all information into the
-# tableSourceConverter.properties of flink-table module.
-################################################################################
-scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java
new file mode 100644
index 0000000..ed92863
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactoryTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_011;
+
+/**
+ * Tests for {@link Kafka011JsonTableSourceFactory}.
+ */
+public class Kafka011JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase {
+
+ @Override
+ protected String version() {
+ return CONNECTOR_VERSION_VALUE_011;
+ }
+
+ @Override
+ protected KafkaJsonTableSource.Builder builder() {
+ return Kafka011JsonTableSource.builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
deleted file mode 100644
index 84ac39b..0000000
--- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.table.descriptors.Kafka;
-
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011;
-
-/**
- * Tests for {@link Kafka011JsonTableSourceFactory}.
- */
-public class Kafka011TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
- protected String versionForTest() {
- return KAFKA_VERSION_VALUE_011;
- }
-
- protected KafkaJsonTableSource.Builder builderForTest() {
- return Kafka011JsonTableSource.builder();
- }
-
- @Override
- protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
- // no extra settings
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.8/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/pom.xml b/flink-connectors/flink-connector-kafka-0.8/pom.xml
index a58591e..43a9dac 100644
--- a/flink-connectors/flink-connector-kafka-0.8/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.8/pom.xml
@@ -204,6 +204,14 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
index e4e5096..2da805a 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
@@ -18,12 +18,13 @@
package org.apache.flink.streaming.connectors.kafka;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_08;
/**
* Factory for creating configured instances of {@link Kafka08JsonTableSource}.
*/
public class Kafka08JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+
@Override
protected KafkaJsonTableSource.Builder createBuilder() {
return new Kafka08JsonTableSource.Builder();
@@ -31,6 +32,6 @@ public class Kafka08JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
@Override
protected String kafkaVersion() {
- return KAFKA_VERSION_VALUE_08;
+ return CONNECTOR_VERSION_VALUE_08;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
deleted file mode 100644
index 5409b49..0000000
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-################################################################################
-# The config file is used to specify the packages of current module where
-# to find TableSourceConverter implementation class annotated with TableType.
-# If there are multiple packages to scan, put those packages together into a
-# string separated with ',', for example, org.package1,org.package2.
-# Please notice:
-# It's better to have a tableSourceConverter.properties in each connector Module
-# which offers converters instead of put all information into the
-# tableSourceConverter.properties of flink-table module.
-################################################################################
-scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java
new file mode 100644
index 0000000..0238b2b
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactoryTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_08;
+
+/**
+ * Tests for {@link Kafka08JsonTableSourceFactory}.
+ */
+public class Kafka08JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase {
+
+ @Override
+ protected String version() {
+ return CONNECTOR_VERSION_VALUE_08;
+ }
+
+ @Override
+ protected KafkaJsonTableSource.Builder builder() {
+ return Kafka08JsonTableSource.builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
deleted file mode 100644
index a2edc09..0000000
--- a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.table.descriptors.Kafka;
-
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08;
-
-/**
- * Tests for {@link Kafka08JsonTableSourceFactory}.
- */
-public class Kafka08TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
- protected String versionForTest() {
- return KAFKA_VERSION_VALUE_08;
- }
-
- protected KafkaJsonTableSource.Builder builderForTest() {
- return Kafka08JsonTableSource.builder();
- }
-
- @Override
- protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
- builder.getKafkaProps().put("zookeeper.connect", "localhost:1111");
- kafka.zookeeperConnect("localhost:1111");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.9/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/pom.xml b/flink-connectors/flink-connector-kafka-0.9/pom.xml
index d07cb5a..1999a8d 100644
--- a/flink-connectors/flink-connector-kafka-0.9/pom.xml
+++ b/flink-connectors/flink-connector-kafka-0.9/pom.xml
@@ -174,6 +174,14 @@ under the License.
</dependency>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<version>${minikdc.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
index bbda4ae..9207426 100644
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
@@ -18,12 +18,13 @@
package org.apache.flink.streaming.connectors.kafka;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_09;
/**
* Factory for creating configured instances of {@link Kafka09JsonTableSource}.
*/
public class Kafka09JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+
@Override
protected KafkaJsonTableSource.Builder createBuilder() {
return new Kafka09JsonTableSource.Builder();
@@ -31,6 +32,6 @@ public class Kafka09JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
@Override
protected String kafkaVersion() {
- return KAFKA_VERSION_VALUE_09;
+ return CONNECTOR_VERSION_VALUE_09;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
deleted file mode 100644
index 5409b49..0000000
--- a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-################################################################################
-# The config file is used to specify the packages of current module where
-# to find TableSourceConverter implementation class annotated with TableType.
-# If there are multiple packages to scan, put those packages together into a
-# string separated with ',', for example, org.package1,org.package2.
-# Please notice:
-# It's better to have a tableSourceConverter.properties in each connector Module
-# which offers converters instead of put all information into the
-# tableSourceConverter.properties of flink-table module.
-################################################################################
-scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java
new file mode 100644
index 0000000..dd545e9
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactoryTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_VERSION_VALUE_09;
+
+/**
+ * Factory for creating configured instances of {@link Kafka09JsonTableSource}.
+ */
+public class Kafka09JsonTableSourceFactoryTest extends KafkaJsonTableSourceFactoryTestBase {
+
+ @Override
+ protected String version() {
+ return CONNECTOR_VERSION_VALUE_09;
+ }
+
+ @Override
+ protected KafkaJsonTableSource.Builder builder() {
+ return Kafka09JsonTableSource.builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
deleted file mode 100644
index fc85ea7..0000000
--- a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.table.descriptors.Kafka;
-
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09;
-
-/**
- * Factory for creating configured instances of {@link Kafka09JsonTableSource}.
- */
-public class Kafka09TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
- protected String versionForTest() {
- return KAFKA_VERSION_VALUE_09;
- }
-
- protected KafkaJsonTableSource.Builder builderForTest() {
- return Kafka09JsonTableSource.builder();
- }
-
- @Override
- protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
- // no extra settings
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index 2ccaa2e..cea3896 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -199,23 +199,17 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minikdc</artifactId>
- <version>${minikdc.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-scala_${scala.binary.version}</artifactId>
+ <artifactId>flink-table_${scala.binary.version}</artifactId>
<version>${project.version}</version>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <version>${minikdc.version}</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
index d2dafe7..b2bb8ff 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
@@ -84,23 +84,29 @@ public abstract class KafkaJsonTableSource extends KafkaTableSource implements D
@Override
public String explainSource() {
- return "KafkaJSONTableSource";
+ return "KafkaJsonTableSource";
}
@Override
- public boolean equals(Object other) {
- if (super.equals(other)) {
- KafkaJsonTableSource otherSource = (KafkaJsonTableSource) other;
- return Objects.equals(failOnMissingField, otherSource.failOnMissingField)
- && Objects.equals(jsonSchema, otherSource.jsonSchema)
- && Objects.equals(fieldMapping, otherSource.fieldMapping);
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
}
- return false;
+ if (!(o instanceof KafkaJsonTableSource)) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ KafkaJsonTableSource that = (KafkaJsonTableSource) o;
+ return failOnMissingField == that.failOnMissingField &&
+ Objects.equals(jsonSchema, that.jsonSchema) &&
+ Objects.equals(fieldMapping, that.fieldMapping);
}
@Override
public int hashCode() {
- return 31 * super.hashCode() + Objects.hash(failOnMissingField, jsonSchema, fieldMapping);
+ return Objects.hash(super.hashCode(), jsonSchema, fieldMapping, failOnMissingField);
}
//////// SETTERS FOR OPTIONAL PARAMETERS
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
index 918b833..2897314 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
@@ -21,65 +21,74 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.formats.json.JsonSchemaConverter;
import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
+import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.descriptors.DescriptorProperties;
import org.apache.flink.table.descriptors.JsonValidator;
import org.apache.flink.table.descriptors.KafkaValidator;
import org.apache.flink.table.descriptors.SchemaValidator;
+import org.apache.flink.table.sources.RowtimeAttributeDescriptor;
import org.apache.flink.table.sources.TableSource;
import org.apache.flink.table.sources.TableSourceFactory;
import org.apache.flink.types.Row;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION;
import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE;
import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION;
+import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_DERIVE_SCHEMA;
+import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_PROPERTY_VERSION;
import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_TYPE;
-import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION;
import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD;
-import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA_STRING;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_JSON_SCHEMA;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA;
import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE;
-import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS;
-import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE;
-import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID;
-import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION;
-import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET;
-import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION;
-import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS;
-import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE;
-import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_EARLIEST;
-import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_GROUP_OFFSETS;
-import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_LATEST;
-import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_SPECIFIC_OFFSETS;
-import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD;
-import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING;
-import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC;
-import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT;
-import static org.apache.flink.table.descriptors.SchemaValidator.PROCTIME;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_KEY;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_VALUE;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_OFFSET;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_PARTITION;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_STARTUP_MODE;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TOPIC;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE_KAFKA;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_CLASS;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_FROM;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_SERIALIZED;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_TIMESTAMPS_TYPE;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_CLASS;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_DELAY;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_SERIALIZED;
+import static org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_WATERMARKS_TYPE;
import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA;
-import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION;
-
-import scala.Option;
-import scala.collection.JavaConversions;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_FROM;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_NAME;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_PROCTIME;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_TYPE;
/**
* Factory for creating configured instances of {@link KafkaJsonTableSource}.
*/
public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory<Row> {
+
@Override
public Map<String, String> requiredContext() {
Map<String, String> context = new HashMap<>();
- context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE); // kafka connector
- context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE()); // Json format
- context.put(KAFKA_VERSION, kafkaVersion()); // for different implementations
- context.put(CONNECTOR_VERSION(), "1");
- context.put(FORMAT_VERSION(), "1");
- context.put(SCHEMA_VERSION(), "1");
+ context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE_KAFKA); // kafka
+ context.put(CONNECTOR_VERSION(), kafkaVersion());
+
+ context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE); // json format
+
+ context.put(CONNECTOR_PROPERTY_VERSION(), "1"); // backwards compatibility
+ context.put(FORMAT_PROPERTY_VERSION(), "1");
+
return context;
}
@@ -88,124 +97,137 @@ public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory<
List<String> properties = new ArrayList<>();
// kafka
- properties.add(KAFKA_VERSION);
- properties.add(BOOTSTRAP_SERVERS);
- properties.add(GROUP_ID);
- properties.add(ZOOKEEPER_CONNECT);
- properties.add(TOPIC);
- properties.add(STARTUP_MODE);
- properties.add(SPECIFIC_OFFSETS + ".#." + PARTITION);
- properties.add(SPECIFIC_OFFSETS + ".#." + OFFSET);
+ properties.add(CONNECTOR_TOPIC);
+ properties.add(CONNECTOR_PROPERTIES);
+ properties.add(CONNECTOR_PROPERTIES + ".#." + CONNECTOR_PROPERTIES_KEY);
+ properties.add(CONNECTOR_PROPERTIES + ".#." + CONNECTOR_PROPERTIES_VALUE);
+ properties.add(CONNECTOR_STARTUP_MODE);
+ properties.add(CONNECTOR_SPECIFIC_OFFSETS + ".#." + CONNECTOR_SPECIFIC_OFFSETS_PARTITION);
+ properties.add(CONNECTOR_SPECIFIC_OFFSETS + ".#." + CONNECTOR_SPECIFIC_OFFSETS_OFFSET);
// json format
- properties.add(FORMAT_SCHEMA_STRING());
- properties.add(FORMAT_FAIL_ON_MISSING_FIELD());
-
- // table json mapping
- properties.add(TABLE_JSON_MAPPING + ".#." + TABLE_FIELD);
- properties.add(TABLE_JSON_MAPPING + ".#." + JSON_FIELD);
+ properties.add(FORMAT_JSON_SCHEMA);
+ properties.add(FORMAT_SCHEMA);
+ properties.add(FORMAT_FAIL_ON_MISSING_FIELD);
+ properties.add(FORMAT_DERIVE_SCHEMA());
// schema
- properties.add(SCHEMA() + ".#." + DescriptorProperties.TYPE());
- properties.add(SCHEMA() + ".#." + DescriptorProperties.NAME());
+ properties.add(SCHEMA() + ".#." + SCHEMA_TYPE());
+ properties.add(SCHEMA() + ".#." + SCHEMA_NAME());
+ properties.add(SCHEMA() + ".#." + SCHEMA_FROM());
// time attributes
- properties.add(SCHEMA() + ".#." + PROCTIME());
-// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_CLASS());
-// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_TYPE());
+ properties.add(SCHEMA() + ".#." + SCHEMA_PROCTIME());
+ properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_TYPE());
+ properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_FROM());
+ properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_CLASS());
+ properties.add(SCHEMA() + ".#." + ROWTIME_TIMESTAMPS_SERIALIZED());
+ properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_TYPE());
+ properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_CLASS());
+ properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_SERIALIZED());
+ properties.add(SCHEMA() + ".#." + ROWTIME_WATERMARKS_DELAY());
return properties;
}
@Override
public TableSource<Row> create(Map<String, String> properties) {
- DescriptorProperties params = new DescriptorProperties(true);
+ final DescriptorProperties params = new DescriptorProperties(true);
params.putProperties(properties);
// validate
+ new SchemaValidator(true).validate(params);
new KafkaValidator().validate(params);
new JsonValidator().validate(params);
- new SchemaValidator(true).validate(params);
// build
- KafkaJsonTableSource.Builder builder = createBuilder();
- Properties kafkaProps = new Properties();
-
- // Set the required parameters.
- String topic = params.getString(TOPIC).get();
- TableSchema tableSchema = params.getTableSchema(SCHEMA()).get();
-
- kafkaProps.put(BOOTSTRAP_SERVERS, params.getString(BOOTSTRAP_SERVERS).get());
- kafkaProps.put(GROUP_ID, params.getString(GROUP_ID).get());
-
- // Set the zookeeper connect for kafka 0.8.
- Option<String> zkConnect = params.getString(ZOOKEEPER_CONNECT);
- if (zkConnect.isDefined()) {
- kafkaProps.put(ZOOKEEPER_CONNECT, zkConnect.get());
- }
-
- builder.withKafkaProperties(kafkaProps).forTopic(topic).withSchema(tableSchema);
-
- // Set the startup mode.
- String startupMode = params.getString(STARTUP_MODE).get();
- if (null != startupMode) {
- switch (startupMode) {
- case STARTUP_MODE_VALUE_EARLIEST:
+ final KafkaJsonTableSource.Builder builder = createBuilder();
+
+ // topic
+ final String topic = params.getString(CONNECTOR_TOPIC);
+ builder.forTopic(topic);
+
+ // properties
+ final Properties props = new Properties();
+ final List<Map<String, String>> propsList = params.getFixedIndexedProperties(
+ CONNECTOR_PROPERTIES,
+ Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE));
+ propsList.forEach(kv -> props.put(
+ params.getString(kv.get(CONNECTOR_PROPERTIES_KEY)),
+ params.getString(kv.get(CONNECTOR_PROPERTIES_VALUE))
+ ));
+ builder.withKafkaProperties(props);
+
+ // startup mode
+ params
+ .getOptionalString(CONNECTOR_STARTUP_MODE)
+ .ifPresent(startupMode -> {
+ switch (startupMode) {
+
+ case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_EARLIEST:
builder.fromEarliest();
break;
- case STARTUP_MODE_VALUE_LATEST:
+
+ case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_LATEST:
builder.fromLatest();
break;
- case STARTUP_MODE_VALUE_GROUP_OFFSETS:
+
+ case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS:
builder.fromGroupOffsets();
break;
- case STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
- Map<String, String> partitions = JavaConversions.
- mapAsJavaMap(params.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION));
- Map<KafkaTopicPartition, Long> offsetMap = new HashMap<>();
- for (int i = 0; i < partitions.size(); i++) {
- offsetMap.put(
- new KafkaTopicPartition(
- topic,
- Integer.valueOf(params.getString(
- SPECIFIC_OFFSETS + "" + "." + i + "." + PARTITION).get())),
- Long.valueOf(params.getString(
- SPECIFIC_OFFSETS + "" + "." + i + "." + OFFSET).get()));
- }
+
+ case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
+ final Map<KafkaTopicPartition, Long> offsetMap = new HashMap<>();
+
+ final List<Map<String, String>> offsetList = params.getFixedIndexedProperties(
+ CONNECTOR_SPECIFIC_OFFSETS,
+ Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
+ offsetList.forEach(kv -> {
+ final int partition = params.getInt(kv.get(CONNECTOR_SPECIFIC_OFFSETS_PARTITION));
+ final long offset = params.getLong(kv.get(CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
+ final KafkaTopicPartition topicPartition = new KafkaTopicPartition(topic, partition);
+ offsetMap.put(topicPartition, offset);
+ });
builder.fromSpecificOffsets(offsetMap);
break;
- }
- }
-
- // Set whether fail on missing JSON field.
- Option<String> failOnMissing = params.getString(FORMAT_FAIL_ON_MISSING_FIELD());
- if (failOnMissing.isDefined()) {
- builder.failOnMissingField(Boolean.valueOf(failOnMissing.get()));
+ }
+ });
+
+ // missing field
+ params.getOptionalBoolean(FORMAT_FAIL_ON_MISSING_FIELD).ifPresent(builder::failOnMissingField);
+
+ // json schema
+ final TableSchema formatSchema;
+ if (params.containsKey(FORMAT_SCHEMA)) {
+ final TypeInformation<?> info = params.getType(FORMAT_SCHEMA);
+ formatSchema = TableSchema.fromTypeInfo(info);
+ } else if (params.containsKey(FORMAT_JSON_SCHEMA)) {
+ final TypeInformation<?> info = JsonSchemaConverter.convert(params.getString(FORMAT_JSON_SCHEMA));
+ formatSchema = TableSchema.fromTypeInfo(info);
+ } else {
+ formatSchema = SchemaValidator.deriveFormatFields(params);
}
+ builder.forJsonSchema(formatSchema);
- // Set the JSON schema.
- Option<String> jsonSchema = params.getString(FORMAT_SCHEMA_STRING());
- if (jsonSchema.isDefined()) {
- TypeInformation jsonSchemaType = JsonSchemaConverter.convert(jsonSchema.get());
- builder.forJsonSchema(TableSchema.fromTypeInfo(jsonSchemaType));
- }
-
- // Set the table => JSON fields mapping.
- Map<String, String> mappingTableFields = JavaConversions.
- mapAsJavaMap(params.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD));
-
- if (!mappingTableFields.isEmpty()) {
- Map<String, String> tableJsonMapping = new HashMap<>();
- for (int i = 0; i < mappingTableFields.size(); i++) {
- tableJsonMapping.put(params.getString(TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD).get(),
- params.getString(TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD).get()
- );
- }
- builder.withTableToJsonMapping(tableJsonMapping);
+ // schema
+ final TableSchema schema = params.getTableSchema(SCHEMA());
+ builder.withSchema(schema);
+
+ // proctime
+ SchemaValidator.deriveProctimeAttribute(params).ifPresent(builder::withProctimeAttribute);
+
+ // rowtime
+ final List<RowtimeAttributeDescriptor> descriptors = SchemaValidator.deriveRowtimeAttributes(params);
+ if (descriptors.size() > 1) {
+ throw new TableException("More than one rowtime attribute is not supported yet.");
+ } else if (descriptors.size() == 1) {
+ final RowtimeAttributeDescriptor desc = descriptors.get(0);
+ builder.withRowtimeAttribute(desc.getAttributeName(), desc.getTimestampExtractor(), desc.getWatermarkStrategy());
}
- // Set the time attributes.
- setTimeAttributes(tableSchema, params, builder);
+ // field mapping
+ final Map<String, String> mapping = SchemaValidator.deriveFieldMapping(params, Optional.of(formatSchema));
+ builder.withTableToJsonMapping(mapping);
return builder.build();
}
@@ -213,15 +235,4 @@ public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory<
protected abstract KafkaJsonTableSource.Builder createBuilder();
protected abstract String kafkaVersion();
-
- private void setTimeAttributes(TableSchema schema, DescriptorProperties params, KafkaJsonTableSource.Builder builder) {
- // TODO to deal with rowtime fields
- Option<String> proctimeField;
- for (int i = 0; i < schema.getColumnNum(); i++) {
- proctimeField = params.getString(SCHEMA() + "." + i + "." + PROCTIME());
- if (proctimeField.isDefined()) {
- builder.withProctimeAttribute(schema.getColumnName(i).get());
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
index 9ce3b8e..134c483 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
@@ -141,31 +141,27 @@ public abstract class KafkaTableSource
@Override
public boolean equals(Object o) {
- if (!o.getClass().equals(this.getClass())) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof KafkaTableSource)) {
return false;
}
- KafkaTableSource other = (KafkaTableSource) o;
- return Objects.equals(topic, other.topic)
- && Objects.equals(schema, other.schema)
- && Objects.equals(properties, other.properties)
- && Objects.equals(proctimeAttribute, other.proctimeAttribute)
- && Objects.equals(returnType, other.returnType)
- && Objects.equals(rowtimeAttributeDescriptors, other.rowtimeAttributeDescriptors)
- && Objects.equals(specificStartupOffsets, other.specificStartupOffsets)
- && Objects.equals(startupMode, other.startupMode);
+ KafkaTableSource that = (KafkaTableSource) o;
+ return Objects.equals(schema, that.schema) &&
+ Objects.equals(topic, that.topic) &&
+ Objects.equals(properties, that.properties) &&
+ Objects.equals(returnType, that.returnType) &&
+ Objects.equals(proctimeAttribute, that.proctimeAttribute) &&
+ Objects.equals(rowtimeAttributeDescriptors, that.rowtimeAttributeDescriptors) &&
+ startupMode == that.startupMode &&
+ Objects.equals(specificStartupOffsets, that.specificStartupOffsets);
}
@Override
public int hashCode() {
- return Objects.hash(
- topic,
- schema,
- properties,
- proctimeAttribute,
- returnType,
- rowtimeAttributeDescriptors,
- specificStartupOffsets,
- startupMode);
+ return Objects.hash(schema, topic, properties, returnType,
+ proctimeAttribute, rowtimeAttributeDescriptors, startupMode, specificStartupOffsets);
}
/**
@@ -211,9 +207,9 @@ public abstract class KafkaTableSource
// validate that field exists and is of correct type
Option<TypeInformation<?>> tpe = schema.getType(proctimeAttribute);
if (tpe.isEmpty()) {
- throw new ValidationException("Processing time attribute " + proctimeAttribute + " is not present in TableSchema.");
+ throw new ValidationException("Processing time attribute '" + proctimeAttribute + "' is not present in TableSchema.");
} else if (tpe.get() != Types.SQL_TIMESTAMP()) {
- throw new ValidationException("Processing time attribute " + proctimeAttribute + " is not of type SQL_TIMESTAMP.");
+ throw new ValidationException("Processing time attribute '" + proctimeAttribute + "' is not of type SQL_TIMESTAMP.");
}
}
this.proctimeAttribute = proctimeAttribute;
@@ -230,9 +226,9 @@ public abstract class KafkaTableSource
String rowtimeAttribute = desc.getAttributeName();
Option<TypeInformation<?>> tpe = schema.getType(rowtimeAttribute);
if (tpe.isEmpty()) {
- throw new ValidationException("Rowtime attribute " + rowtimeAttribute + " is not present in TableSchema.");
+ throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not present in TableSchema.");
} else if (tpe.get() != Types.SQL_TIMESTAMP()) {
- throw new ValidationException("Rowtime attribute " + rowtimeAttribute + " is not of type SQL_TIMESTAMP.");
+ throw new ValidationException("Rowtime attribute '" + rowtimeAttribute + "' is not of type SQL_TIMESTAMP.");
}
}
this.rowtimeAttributeDescriptors = rowtimeAttributeDescriptors;
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
index 4733f6e..4535958 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
@@ -18,182 +18,199 @@
package org.apache.flink.table.descriptors;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase;
import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
-
-import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS;
-import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE;
-import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID;
-import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD;
-import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION;
-import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET;
-import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION;
-import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS;
-import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD;
-import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING;
-import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC;
-import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT;
+import org.apache.flink.util.Preconditions;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-
-import scala.collection.JavaConversions;
-import scala.collection.Seq;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_KEY;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_PROPERTIES_VALUE;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_OFFSET;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_SPECIFIC_OFFSETS_PARTITION;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_STARTUP_MODE;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TOPIC;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE_KAFKA;
/**
- * Connector descriptor for the kafka message queue.
+ * Connector descriptor for the Apache Kafka message queue.
*/
public class Kafka extends ConnectorDescriptor {
- private Optional<String> version = Optional.empty();
- private Optional<String> bootstrapServers = Optional.empty();
- private Optional<String> groupId = Optional.empty();
- private Optional<String> topic = Optional.empty();
- private Optional<String> zookeeperConnect = Optional.empty();
- private Optional<Map<String, String>> tableJsonMapping = Optional.empty();
-
- private Optional<StartupMode> startupMode = Optional.empty();
- private Optional<Map<Integer, Long>> specificOffsets = Optional.empty();
+ private String version;
+ private String topic;
+ private StartupMode startupMode;
+ private Map<Integer, Long> specificOffsets;
+ private Map<String, String> kafkaProperties;
+ /**
+ * Connector descriptor for the Apache Kafka message queue.
+ */
public Kafka() {
- super(CONNECTOR_TYPE_VALUE, 1);
+ super(CONNECTOR_TYPE_VALUE_KAFKA, 1, true);
}
/**
- * Sets the kafka version.
+ * Sets the Kafka version to be used.
*
- * @param version
- * Could be {@link KafkaValidator#KAFKA_VERSION_VALUE_011},
- * {@link KafkaValidator#KAFKA_VERSION_VALUE_010},
- * {@link KafkaValidator#KAFKA_VERSION_VALUE_09},
- * or {@link KafkaValidator#KAFKA_VERSION_VALUE_08}.
+ * @param version Kafka version. E.g., "0.8", "0.11", etc.
*/
public Kafka version(String version) {
- this.version = Optional.of(version);
+ Preconditions.checkNotNull(version);
+ this.version = version;
return this;
}
/**
- * Sets the bootstrap servers for kafka.
+ * Sets the topic from which the table is read.
+ *
+ * @param topic The topic from which the table is read.
*/
- public Kafka bootstrapServers(String bootstrapServers) {
- this.bootstrapServers = Optional.of(bootstrapServers);
+ public Kafka topic(String topic) {
+ Preconditions.checkNotNull(topic);
+ this.topic = topic;
return this;
}
/**
- * Sets the consumer group id.
+ * Sets the configuration properties for the Kafka consumer. Resets previously set properties.
+ *
+ * @param properties The configuration properties for the Kafka consumer.
*/
- public Kafka groupId(String groupId) {
- this.groupId = Optional.of(groupId);
+ public Kafka properties(Properties properties) {
+ Preconditions.checkNotNull(properties);
+ if (this.kafkaProperties == null) {
+ this.kafkaProperties = new HashMap<>();
+ }
+ this.kafkaProperties.clear();
+ properties.forEach((k, v) -> this.kafkaProperties.put((String) k, (String) v));
return this;
}
/**
- * Sets the topic to consume.
+ * Adds a configuration properties for the Kafka consumer.
+ *
+ * @param key property key for the Kafka consumer
+ * @param value property value for the Kafka consumer
*/
- public Kafka topic(String topic) {
- this.topic = Optional.of(topic);
+ public Kafka property(String key, String value) {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(value);
+ if (this.kafkaProperties == null) {
+ this.kafkaProperties = new HashMap<>();
+ }
+ kafkaProperties.put(key, value);
return this;
}
/**
- * Sets the startup mode.
+ * Configures to start reading from the earliest offset for all partitions.
+ *
+ * @see FlinkKafkaConsumerBase#setStartFromEarliest()
*/
- public Kafka startupMode(StartupMode startupMode) {
- this.startupMode = Optional.of(startupMode);
+ public Kafka startFromEarliest() {
+ this.startupMode = StartupMode.EARLIEST;
+ this.specificOffsets = null;
return this;
}
/**
- * Sets the zookeeper hosts. Only required by kafka 0.8.
+ * Configures to start reading from the latest offset for all partitions.
+ *
+ * @see FlinkKafkaConsumerBase#setStartFromLatest()
*/
- public Kafka zookeeperConnect(String zookeeperConnect) {
- this.zookeeperConnect = Optional.of(zookeeperConnect);
+ public Kafka startFromLatest() {
+ this.startupMode = StartupMode.LATEST;
+ this.specificOffsets = null;
return this;
}
/**
- * Sets the consume offsets for the topic set with {@link Kafka#topic(String)}.
- * Only works in {@link StartupMode#SPECIFIC_OFFSETS} mode.
+ * Configures to start reading from any committed group offsets found in Zookeeper / Kafka brokers.
+ *
+ * @see FlinkKafkaConsumerBase#setStartFromGroupOffsets()
*/
- public Kafka specificOffsets(Map<Integer, Long> specificOffsets) {
- this.specificOffsets = Optional.of(specificOffsets);
+ public Kafka startFromGroupOffsets() {
+ this.startupMode = StartupMode.GROUP_OFFSETS;
+ this.specificOffsets = null;
return this;
}
/**
- * Sets the mapping from logical table schema to json schema.
+ * Configures to start reading partitions from specific offsets, set independently for each partition.
+ * Resets previously set offsets.
+ *
+ * @param specificOffsets the specified offsets for partitions
+ * @see FlinkKafkaConsumerBase#setStartFromSpecificOffsets(Map)
*/
- public Kafka tableJsonMapping(Map<String, String> jsonTableMapping) {
- this.tableJsonMapping = Optional.of(jsonTableMapping);
+ public Kafka startFromSpecificOffsets(Map<Integer, Long> specificOffsets) {
+ this.startupMode = StartupMode.SPECIFIC_OFFSETS;
+ this.specificOffsets = Preconditions.checkNotNull(specificOffsets);
return this;
}
+ /**
+ * Configures to start reading partitions from specific offsets and specifies the given offset for
+ * the given partition.
+ *
+ * @param partition partition index
+ * @param specificOffset partition offset to start reading from
+ * @see FlinkKafkaConsumerBase#setStartFromSpecificOffsets(Map)
+ */
+ public Kafka startFromSpecificOffset(int partition, long specificOffset) {
+ this.startupMode = StartupMode.SPECIFIC_OFFSETS;
+ if (this.specificOffsets == null) {
+ this.specificOffsets = new HashMap<>();
+ }
+ this.specificOffsets.put(partition, specificOffset);
+ return this;
+ }
+
+ /**
+ * Internal method for connector properties conversion.
+ */
@Override
public void addConnectorProperties(DescriptorProperties properties) {
- if (version.isPresent()) {
- properties.putString(KAFKA_VERSION, version.get());
- }
- if (bootstrapServers.isPresent()) {
- properties.putString(BOOTSTRAP_SERVERS, bootstrapServers.get());
- }
- if (groupId.isPresent()) {
- properties.putString(GROUP_ID, groupId.get());
- }
- if (topic.isPresent()) {
- properties.putString(TOPIC, topic.get());
+ if (version != null) {
+ properties.putString(CONNECTOR_VERSION(), version);
}
- if (zookeeperConnect.isPresent()) {
- properties.putString(ZOOKEEPER_CONNECT, zookeeperConnect.get());
+
+ if (topic != null) {
+ properties.putString(CONNECTOR_TOPIC, topic);
}
- if (startupMode.isPresent()) {
- Map<String, String> map = KafkaValidator.normalizeStartupMode(startupMode.get());
- for (Map.Entry<String, String> entry : map.entrySet()) {
- properties.putString(entry.getKey(), entry.getValue());
- }
+
+ if (startupMode != null) {
+ properties.putString(CONNECTOR_STARTUP_MODE, KafkaValidator.normalizeStartupMode(startupMode));
}
- if (specificOffsets.isPresent()) {
- List<String> propertyKeys = new ArrayList<>();
- propertyKeys.add(PARTITION);
- propertyKeys.add(OFFSET);
-
- List<Seq<String>> propertyValues = new ArrayList<>(specificOffsets.get().size());
- for (Map.Entry<Integer, Long> entry : specificOffsets.get().entrySet()) {
- List<String> partitionOffset = new ArrayList<>(2);
- partitionOffset.add(entry.getKey().toString());
- partitionOffset.add(entry.getValue().toString());
- propertyValues.add(JavaConversions.asScalaBuffer(partitionOffset).toSeq());
+
+ if (specificOffsets != null) {
+ final List<List<String>> values = new ArrayList<>();
+ for (Map.Entry<Integer, Long> specificOffset : specificOffsets.entrySet()) {
+ values.add(Arrays.asList(specificOffset.getKey().toString(), specificOffset.getValue().toString()));
}
properties.putIndexedFixedProperties(
- SPECIFIC_OFFSETS,
- JavaConversions.asScalaBuffer(propertyKeys).toSeq(),
- JavaConversions.asScalaBuffer(propertyValues).toSeq()
- );
+ CONNECTOR_SPECIFIC_OFFSETS,
+ Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET),
+ values);
}
- if (tableJsonMapping.isPresent()) {
- List<String> propertyKeys = new ArrayList<>();
- propertyKeys.add(TABLE_FIELD);
- propertyKeys.add(JSON_FIELD);
-
- List<Seq<String>> mappingFields = new ArrayList<>(tableJsonMapping.get().size());
- for (Map.Entry<String, String> entry : tableJsonMapping.get().entrySet()) {
- List<String> singleMapping = new ArrayList<>(2);
- singleMapping.add(entry.getKey());
- singleMapping.add(entry.getValue());
- mappingFields.add(JavaConversions.asScalaBuffer(singleMapping).toSeq());
- }
+
+ if (kafkaProperties != null) {
properties.putIndexedFixedProperties(
- TABLE_JSON_MAPPING,
- JavaConversions.asScalaBuffer(propertyKeys).toSeq(),
- JavaConversions.asScalaBuffer(mappingFields).toSeq()
- );
+ CONNECTOR_PROPERTIES,
+ Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE),
+ this.kafkaProperties.entrySet().stream()
+ .map(e -> Arrays.asList(e.getKey(), e.getValue()))
+ .collect(Collectors.toList())
+ );
}
}
-
- @Override
- public boolean needsFormat() {
- return true;
- }
}
[5/5] flink git commit: [FLINK-8538][table]Add a Kafka table source
factory with JSON format support
Posted by tw...@apache.org.
[FLINK-8538][table]Add a Kafka table source factory with JSON format support
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1d26062d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1d26062d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1d26062d
Branch: refs/heads/release-1.5
Commit: 1d26062de130c05fdbe7701b55766b4a8d433418
Parents: a269f85
Author: Xingcan Cui <xi...@gmail.com>
Authored: Mon Feb 12 18:11:36 2018 +0800
Committer: Timo Walther <tw...@apache.org>
Committed: Tue Feb 27 20:23:00 2018 +0100
----------------------------------------------------------------------
.../kafka/Kafka010JsonTableSourceFactory.java | 36 +++
...pache.flink.table.sources.TableSourceFactory | 16 ++
.../resources/tableSourceConverter.properties | 29 +++
.../kafka/Kafka010TableSourceFactoryTest.java | 41 ++++
.../kafka/Kafka011JsonTableSourceFactory.java | 36 +++
...pache.flink.table.sources.TableSourceFactory | 16 ++
.../resources/tableSourceConverter.properties | 29 +++
.../kafka/Kafka011TableSourceFactoryTest.java | 41 ++++
.../kafka/Kafka08JsonTableSourceFactory.java | 36 +++
...pache.flink.table.sources.TableSourceFactory | 16 ++
.../resources/tableSourceConverter.properties | 29 +++
.../kafka/Kafka08TableSourceFactoryTest.java | 42 ++++
.../kafka/Kafka09JsonTableSourceFactory.java | 36 +++
...pache.flink.table.sources.TableSourceFactory | 16 ++
.../resources/tableSourceConverter.properties | 29 +++
.../kafka/Kafka09TableSourceFactoryTest.java | 41 ++++
.../flink-connector-kafka-base/pom.xml | 14 ++
.../connectors/kafka/KafkaJsonTableSource.java | 17 ++
.../kafka/KafkaJsonTableSourceFactory.java | 227 +++++++++++++++++++
.../connectors/kafka/KafkaTableSource.java | 30 +++
.../apache/flink/table/descriptors/Kafka.java | 199 ++++++++++++++++
.../flink/table/descriptors/KafkaValidator.java | 193 ++++++++++++++++
.../KafkaJsonTableFromDescriptorTestBase.java | 127 +++++++++++
.../src/test/resources/kafka-json-schema.json | 35 +++
.../apache/flink/table/api/TableSchema.scala | 5 +
25 files changed, 1336 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
new file mode 100644
index 0000000..1d03f6c
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka010JsonTableSourceFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010;
+
+/**
+ * Factory for creating configured instances of {@link Kafka010JsonTableSource}.
+ */
+public class Kafka010JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+ @Override
+ protected KafkaJsonTableSource.Builder createBuilder() {
+ return new Kafka010JsonTableSource.Builder();
+ }
+
+ @Override
+ protected String kafkaVersion() {
+ return KAFKA_VERSION_VALUE_010;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
new file mode 100644
index 0000000..9ef54fc
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.streaming.connectors.kafka.Kafka010JsonTableSourceFactory
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
new file mode 100644
index 0000000..5409b49
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/main/resources/tableSourceConverter.properties
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+################################################################################
+# The config file is used to specify the packages of current module where
+# to find TableSourceConverter implementation class annotated with TableType.
+# If there are multiple packages to scan, put those packages together into a
+# string separated with ',', for example, org.package1,org.package2.
+# Please notice:
+# It's better to have a tableSourceConverter.properties in each connector Module
+# which offers converters instead of put all information into the
+# tableSourceConverter.properties of flink-table module.
+################################################################################
+scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
new file mode 100644
index 0000000..15b89e8
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.10/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka010TableSourceFactoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.table.descriptors.Kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_010;
+
+/**
+ * Tests for {@link Kafka010JsonTableSourceFactory}.
+ */
+public class Kafka010TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
+ protected String versionForTest() {
+ return KAFKA_VERSION_VALUE_010;
+ }
+
+ protected KafkaJsonTableSource.Builder builderForTest() {
+ return Kafka010JsonTableSource.builder();
+ }
+
+ @Override
+ protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
+ // no extra settings
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
new file mode 100644
index 0000000..ca4d6ce
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka011JsonTableSourceFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011;
+
+/**
+ * Factory for creating configured instances of {@link Kafka011JsonTableSource}.
+ */
+public class Kafka011JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+ @Override
+ protected KafkaJsonTableSource.Builder createBuilder() {
+ return new Kafka011JsonTableSource.Builder();
+ }
+
+ @Override
+ protected String kafkaVersion() {
+ return KAFKA_VERSION_VALUE_011;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
new file mode 100644
index 0000000..75135e5
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.streaming.connectors.kafka.Kafka011JsonTableSourceFactory
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
new file mode 100644
index 0000000..5409b49
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.11/src/main/resources/tableSourceConverter.properties
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+################################################################################
+# The config file is used to specify the packages of current module where
+# to find TableSourceConverter implementation class annotated with TableType.
+# If there are multiple packages to scan, put those packages together into a
+# string separated with ',', for example, org.package1,org.package2.
+# Please notice:
+# It's better to have a tableSourceConverter.properties in each connector Module
+# which offers converters instead of put all information into the
+# tableSourceConverter.properties of flink-table module.
+################################################################################
+scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
new file mode 100644
index 0000000..84ac39b
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka011TableSourceFactoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.table.descriptors.Kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_011;
+
+/**
+ * Tests for {@link Kafka011JsonTableSourceFactory}.
+ */
+public class Kafka011TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
+ protected String versionForTest() {
+ return KAFKA_VERSION_VALUE_011;
+ }
+
+ protected KafkaJsonTableSource.Builder builderForTest() {
+ return Kafka011JsonTableSource.builder();
+ }
+
+ @Override
+ protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
+ // no extra settings
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
new file mode 100644
index 0000000..e4e5096
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka08JsonTableSourceFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08;
+
+/**
+ * Factory for creating configured instances of {@link Kafka08JsonTableSource}.
+ */
+public class Kafka08JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+ @Override
+ protected KafkaJsonTableSource.Builder createBuilder() {
+ return new Kafka08JsonTableSource.Builder();
+ }
+
+ @Override
+ protected String kafkaVersion() {
+ return KAFKA_VERSION_VALUE_08;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
new file mode 100644
index 0000000..9092955
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.streaming.connectors.kafka.Kafka08JsonTableSourceFactory
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
new file mode 100644
index 0000000..5409b49
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/resources/tableSourceConverter.properties
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+################################################################################
+# The config file is used to specify the packages of current module where
+# to find TableSourceConverter implementation class annotated with TableType.
+# If there are multiple packages to scan, put those packages together into a
+# string separated with ',', for example, org.package1,org.package2.
+# Please notice:
+# It's better to have a tableSourceConverter.properties in each connector Module
+# which offers converters instead of put all information into the
+# tableSourceConverter.properties of flink-table module.
+################################################################################
+scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
new file mode 100644
index 0000000..a2edc09
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.8/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka08TableSourceFactoryTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.table.descriptors.Kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_08;
+
+/**
+ * Tests for {@link Kafka08JsonTableSourceFactory}.
+ */
+public class Kafka08TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
+ protected String versionForTest() {
+ return KAFKA_VERSION_VALUE_08;
+ }
+
+ protected KafkaJsonTableSource.Builder builderForTest() {
+ return Kafka08JsonTableSource.builder();
+ }
+
+ @Override
+ protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
+ builder.getKafkaProps().put("zookeeper.connect", "localhost:1111");
+ kafka.zookeeperConnect("localhost:1111");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
new file mode 100644
index 0000000..bbda4ae
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/Kafka09JsonTableSourceFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09;
+
+/**
+ * Factory for creating configured instances of {@link Kafka09JsonTableSource}.
+ */
+public class Kafka09JsonTableSourceFactory extends KafkaJsonTableSourceFactory {
+ @Override
+ protected KafkaJsonTableSource.Builder createBuilder() {
+ return new Kafka09JsonTableSource.Builder();
+ }
+
+ @Override
+ protected String kafkaVersion() {
+ return KAFKA_VERSION_VALUE_09;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
new file mode 100644
index 0000000..2f38bd0
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/META-INF/services/org.apache.flink.table.sources.TableSourceFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.streaming.connectors.kafka.Kafka09JsonTableSourceFactory
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
new file mode 100644
index 0000000..5409b49
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/main/resources/tableSourceConverter.properties
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+################################################################################
+# The config file is used to specify the packages of current module where
+# to find TableSourceConverter implementation class annotated with TableType.
+# If there are multiple packages to scan, put those packages together into a
+# string separated with ',', for example, org.package1,org.package2.
+# Please notice:
+# It's better to have a tableSourceConverter.properties in each connector Module
+# which offers converters instead of put all information into the
+# tableSourceConverter.properties of flink-table module.
+################################################################################
+scan.packages=org.apache.flink.streaming.connectors.kafka
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
new file mode 100644
index 0000000..fc85ea7
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09TableSourceFactoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.table.descriptors.Kafka;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION_VALUE_09;
+
+/**
+ * Factory for creating configured instances of {@link Kafka09JsonTableSource}.
+ */
+public class Kafka09TableSourceFactoryTest extends KafkaJsonTableFromDescriptorTestBase {
+ protected String versionForTest() {
+ return KAFKA_VERSION_VALUE_09;
+ }
+
+ protected KafkaJsonTableSource.Builder builderForTest() {
+ return Kafka09JsonTableSource.builder();
+ }
+
+ @Override
+ protected void extraSettings(KafkaTableSource.Builder builder, Kafka kafka) {
+ // no extra settings
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/pom.xml b/flink-connectors/flink-connector-kafka-base/pom.xml
index e7412cf..2ccaa2e 100644
--- a/flink-connectors/flink-connector-kafka-base/pom.xml
+++ b/flink-connectors/flink-connector-kafka-base/pom.xml
@@ -205,6 +205,20 @@ under the License.
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
index f581e89..d2dafe7 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSource.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.sources.DefinedFieldMapping;
import org.apache.flink.table.sources.StreamTableSource;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
/**
@@ -86,6 +87,22 @@ public abstract class KafkaJsonTableSource extends KafkaTableSource implements D
return "KafkaJSONTableSource";
}
+ @Override
+ public boolean equals(Object other) {
+ if (super.equals(other)) {
+ KafkaJsonTableSource otherSource = (KafkaJsonTableSource) other;
+ return Objects.equals(failOnMissingField, otherSource.failOnMissingField)
+ && Objects.equals(jsonSchema, otherSource.jsonSchema)
+ && Objects.equals(fieldMapping, otherSource.fieldMapping);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * super.hashCode() + Objects.hash(failOnMissingField, jsonSchema, fieldMapping);
+ }
+
//////// SETTERS FOR OPTIONAL PARAMETERS
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
new file mode 100644
index 0000000..918b833
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableSourceFactory.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.formats.json.JsonSchemaConverter;
+import org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.descriptors.DescriptorProperties;
+import org.apache.flink.table.descriptors.JsonValidator;
+import org.apache.flink.table.descriptors.KafkaValidator;
+import org.apache.flink.table.descriptors.SchemaValidator;
+import org.apache.flink.table.sources.TableSource;
+import org.apache.flink.table.sources.TableSourceFactory;
+import org.apache.flink.types.Row;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE;
+import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION;
+import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_TYPE;
+import static org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_FAIL_ON_MISSING_FIELD;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_SCHEMA_STRING;
+import static org.apache.flink.table.descriptors.JsonValidator.FORMAT_TYPE_VALUE;
+import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE;
+import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID;
+import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD;
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION;
+import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET;
+import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION;
+import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE;
+import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_EARLIEST;
+import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_GROUP_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_LATEST;
+import static org.apache.flink.table.descriptors.KafkaValidator.STARTUP_MODE_VALUE_SPECIFIC_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD;
+import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING;
+import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC;
+import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT;
+import static org.apache.flink.table.descriptors.SchemaValidator.PROCTIME;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA;
+import static org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION;
+
+import scala.Option;
+import scala.collection.JavaConversions;
+
+/**
+ * Factory for creating configured instances of {@link KafkaJsonTableSource}.
+ */
+public abstract class KafkaJsonTableSourceFactory implements TableSourceFactory<Row> {
+ @Override
+ public Map<String, String> requiredContext() {
+ Map<String, String> context = new HashMap<>();
+ context.put(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE); // kafka connector
+ context.put(FORMAT_TYPE(), FORMAT_TYPE_VALUE()); // Json format
+ context.put(KAFKA_VERSION, kafkaVersion()); // for different implementations
+ context.put(CONNECTOR_VERSION(), "1");
+ context.put(FORMAT_VERSION(), "1");
+ context.put(SCHEMA_VERSION(), "1");
+ return context;
+ }
+
+ @Override
+ public List<String> supportedProperties() {
+ List<String> properties = new ArrayList<>();
+
+ // kafka
+ properties.add(KAFKA_VERSION);
+ properties.add(BOOTSTRAP_SERVERS);
+ properties.add(GROUP_ID);
+ properties.add(ZOOKEEPER_CONNECT);
+ properties.add(TOPIC);
+ properties.add(STARTUP_MODE);
+ properties.add(SPECIFIC_OFFSETS + ".#." + PARTITION);
+ properties.add(SPECIFIC_OFFSETS + ".#." + OFFSET);
+
+ // json format
+ properties.add(FORMAT_SCHEMA_STRING());
+ properties.add(FORMAT_FAIL_ON_MISSING_FIELD());
+
+ // table json mapping
+ properties.add(TABLE_JSON_MAPPING + ".#." + TABLE_FIELD);
+ properties.add(TABLE_JSON_MAPPING + ".#." + JSON_FIELD);
+
+ // schema
+ properties.add(SCHEMA() + ".#." + DescriptorProperties.TYPE());
+ properties.add(SCHEMA() + ".#." + DescriptorProperties.NAME());
+
+ // time attributes
+ properties.add(SCHEMA() + ".#." + PROCTIME());
+// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_CLASS());
+// properties.add(SCHEMA() + ".#." + ROWTIME() + ".#." + TIMESTAMPS_TYPE());
+
+ return properties;
+ }
+
+ @Override
+ public TableSource<Row> create(Map<String, String> properties) {
+ DescriptorProperties params = new DescriptorProperties(true);
+ params.putProperties(properties);
+
+ // validate
+ new KafkaValidator().validate(params);
+ new JsonValidator().validate(params);
+ new SchemaValidator(true).validate(params);
+
+ // build
+ KafkaJsonTableSource.Builder builder = createBuilder();
+ Properties kafkaProps = new Properties();
+
+ // Set the required parameters.
+ String topic = params.getString(TOPIC).get();
+ TableSchema tableSchema = params.getTableSchema(SCHEMA()).get();
+
+ kafkaProps.put(BOOTSTRAP_SERVERS, params.getString(BOOTSTRAP_SERVERS).get());
+ kafkaProps.put(GROUP_ID, params.getString(GROUP_ID).get());
+
+ // Set the zookeeper connect for kafka 0.8.
+ Option<String> zkConnect = params.getString(ZOOKEEPER_CONNECT);
+ if (zkConnect.isDefined()) {
+ kafkaProps.put(ZOOKEEPER_CONNECT, zkConnect.get());
+ }
+
+ builder.withKafkaProperties(kafkaProps).forTopic(topic).withSchema(tableSchema);
+
+ // Set the startup mode.
+ String startupMode = params.getString(STARTUP_MODE).get();
+ if (null != startupMode) {
+ switch (startupMode) {
+ case STARTUP_MODE_VALUE_EARLIEST:
+ builder.fromEarliest();
+ break;
+ case STARTUP_MODE_VALUE_LATEST:
+ builder.fromLatest();
+ break;
+ case STARTUP_MODE_VALUE_GROUP_OFFSETS:
+ builder.fromGroupOffsets();
+ break;
+ case STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
+ Map<String, String> partitions = JavaConversions.
+ mapAsJavaMap(params.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION));
+ Map<KafkaTopicPartition, Long> offsetMap = new HashMap<>();
+ for (int i = 0; i < partitions.size(); i++) {
+ offsetMap.put(
+ new KafkaTopicPartition(
+ topic,
+ Integer.valueOf(params.getString(
+ SPECIFIC_OFFSETS + "" + "." + i + "." + PARTITION).get())),
+ Long.valueOf(params.getString(
+ SPECIFIC_OFFSETS + "" + "." + i + "." + OFFSET).get()));
+ }
+ builder.fromSpecificOffsets(offsetMap);
+ break;
+ }
+ }
+
+ // Set whether fail on missing JSON field.
+ Option<String> failOnMissing = params.getString(FORMAT_FAIL_ON_MISSING_FIELD());
+ if (failOnMissing.isDefined()) {
+ builder.failOnMissingField(Boolean.valueOf(failOnMissing.get()));
+ }
+
+ // Set the JSON schema.
+ Option<String> jsonSchema = params.getString(FORMAT_SCHEMA_STRING());
+ if (jsonSchema.isDefined()) {
+ TypeInformation jsonSchemaType = JsonSchemaConverter.convert(jsonSchema.get());
+ builder.forJsonSchema(TableSchema.fromTypeInfo(jsonSchemaType));
+ }
+
+ // Set the table => JSON fields mapping.
+ Map<String, String> mappingTableFields = JavaConversions.
+ mapAsJavaMap(params.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD));
+
+ if (!mappingTableFields.isEmpty()) {
+ Map<String, String> tableJsonMapping = new HashMap<>();
+ for (int i = 0; i < mappingTableFields.size(); i++) {
+ tableJsonMapping.put(params.getString(TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD).get(),
+ params.getString(TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD).get()
+ );
+ }
+ builder.withTableToJsonMapping(tableJsonMapping);
+ }
+
+ // Set the time attributes.
+ setTimeAttributes(tableSchema, params, builder);
+
+ return builder.build();
+ }
+
+ protected abstract KafkaJsonTableSource.Builder createBuilder();
+
+ protected abstract String kafkaVersion();
+
+ private void setTimeAttributes(TableSchema schema, DescriptorProperties params, KafkaJsonTableSource.Builder builder) {
+ // TODO to deal with rowtime fields
+ Option<String> proctimeField;
+ for (int i = 0; i < schema.getColumnNum(); i++) {
+ proctimeField = params.getString(SCHEMA() + "." + i + "." + PROCTIME());
+ if (proctimeField.isDefined()) {
+ builder.withProctimeAttribute(schema.getColumnName(i).get());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
index d5cda4a..9ce3b8e 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSource.java
@@ -42,6 +42,7 @@ import org.apache.flink.util.Preconditions;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import scala.Option;
@@ -138,6 +139,35 @@ public abstract class KafkaTableSource
return TableConnectorUtil.generateRuntimeName(this.getClass(), schema.getColumnNames());
}
+ @Override
+ public boolean equals(Object o) {
+ if (!o.getClass().equals(this.getClass())) {
+ return false;
+ }
+ KafkaTableSource other = (KafkaTableSource) o;
+ return Objects.equals(topic, other.topic)
+ && Objects.equals(schema, other.schema)
+ && Objects.equals(properties, other.properties)
+ && Objects.equals(proctimeAttribute, other.proctimeAttribute)
+ && Objects.equals(returnType, other.returnType)
+ && Objects.equals(rowtimeAttributeDescriptors, other.rowtimeAttributeDescriptors)
+ && Objects.equals(specificStartupOffsets, other.specificStartupOffsets)
+ && Objects.equals(startupMode, other.startupMode);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ topic,
+ schema,
+ properties,
+ proctimeAttribute,
+ returnType,
+ rowtimeAttributeDescriptors,
+ specificStartupOffsets,
+ startupMode);
+ }
+
/**
* Returns a version-specific Kafka consumer with the start position configured.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
new file mode 100644
index 0000000..4733f6e
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
+
+import static org.apache.flink.table.descriptors.KafkaValidator.BOOTSTRAP_SERVERS;
+import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_VALUE;
+import static org.apache.flink.table.descriptors.KafkaValidator.GROUP_ID;
+import static org.apache.flink.table.descriptors.KafkaValidator.JSON_FIELD;
+import static org.apache.flink.table.descriptors.KafkaValidator.KAFKA_VERSION;
+import static org.apache.flink.table.descriptors.KafkaValidator.OFFSET;
+import static org.apache.flink.table.descriptors.KafkaValidator.PARTITION;
+import static org.apache.flink.table.descriptors.KafkaValidator.SPECIFIC_OFFSETS;
+import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_FIELD;
+import static org.apache.flink.table.descriptors.KafkaValidator.TABLE_JSON_MAPPING;
+import static org.apache.flink.table.descriptors.KafkaValidator.TOPIC;
+import static org.apache.flink.table.descriptors.KafkaValidator.ZOOKEEPER_CONNECT;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+
+/**
+ * Connector descriptor for the kafka message queue.
+ */
+public class Kafka extends ConnectorDescriptor {
+
+ private Optional<String> version = Optional.empty();
+ private Optional<String> bootstrapServers = Optional.empty();
+ private Optional<String> groupId = Optional.empty();
+ private Optional<String> topic = Optional.empty();
+ private Optional<String> zookeeperConnect = Optional.empty();
+ private Optional<Map<String, String>> tableJsonMapping = Optional.empty();
+
+ private Optional<StartupMode> startupMode = Optional.empty();
+ private Optional<Map<Integer, Long>> specificOffsets = Optional.empty();
+
+ public Kafka() {
+ super(CONNECTOR_TYPE_VALUE, 1);
+ }
+
+ /**
+ * Sets the kafka version.
+ *
+ * @param version
+ * Could be {@link KafkaValidator#KAFKA_VERSION_VALUE_011},
+ * {@link KafkaValidator#KAFKA_VERSION_VALUE_010},
+ * {@link KafkaValidator#KAFKA_VERSION_VALUE_09},
+ * or {@link KafkaValidator#KAFKA_VERSION_VALUE_08}.
+ */
+ public Kafka version(String version) {
+ this.version = Optional.of(version);
+ return this;
+ }
+
+ /**
+ * Sets the bootstrap servers for kafka.
+ */
+ public Kafka bootstrapServers(String bootstrapServers) {
+ this.bootstrapServers = Optional.of(bootstrapServers);
+ return this;
+ }
+
+ /**
+ * Sets the consumer group id.
+ */
+ public Kafka groupId(String groupId) {
+ this.groupId = Optional.of(groupId);
+ return this;
+ }
+
+ /**
+ * Sets the topic to consume.
+ */
+ public Kafka topic(String topic) {
+ this.topic = Optional.of(topic);
+ return this;
+ }
+
+ /**
+ * Sets the startup mode.
+ */
+ public Kafka startupMode(StartupMode startupMode) {
+ this.startupMode = Optional.of(startupMode);
+ return this;
+ }
+
+ /**
+ * Sets the zookeeper hosts. Only required by kafka 0.8.
+ */
+ public Kafka zookeeperConnect(String zookeeperConnect) {
+ this.zookeeperConnect = Optional.of(zookeeperConnect);
+ return this;
+ }
+
+ /**
+ * Sets the consume offsets for the topic set with {@link Kafka#topic(String)}.
+ * Only works in {@link StartupMode#SPECIFIC_OFFSETS} mode.
+ */
+ public Kafka specificOffsets(Map<Integer, Long> specificOffsets) {
+ this.specificOffsets = Optional.of(specificOffsets);
+ return this;
+ }
+
+ /**
+ * Sets the mapping from logical table schema to json schema.
+ */
+ public Kafka tableJsonMapping(Map<String, String> jsonTableMapping) {
+ this.tableJsonMapping = Optional.of(jsonTableMapping);
+ return this;
+ }
+
+ @Override
+ public void addConnectorProperties(DescriptorProperties properties) {
+ if (version.isPresent()) {
+ properties.putString(KAFKA_VERSION, version.get());
+ }
+ if (bootstrapServers.isPresent()) {
+ properties.putString(BOOTSTRAP_SERVERS, bootstrapServers.get());
+ }
+ if (groupId.isPresent()) {
+ properties.putString(GROUP_ID, groupId.get());
+ }
+ if (topic.isPresent()) {
+ properties.putString(TOPIC, topic.get());
+ }
+ if (zookeeperConnect.isPresent()) {
+ properties.putString(ZOOKEEPER_CONNECT, zookeeperConnect.get());
+ }
+ if (startupMode.isPresent()) {
+ Map<String, String> map = KafkaValidator.normalizeStartupMode(startupMode.get());
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ properties.putString(entry.getKey(), entry.getValue());
+ }
+ }
+ if (specificOffsets.isPresent()) {
+ List<String> propertyKeys = new ArrayList<>();
+ propertyKeys.add(PARTITION);
+ propertyKeys.add(OFFSET);
+
+ List<Seq<String>> propertyValues = new ArrayList<>(specificOffsets.get().size());
+ for (Map.Entry<Integer, Long> entry : specificOffsets.get().entrySet()) {
+ List<String> partitionOffset = new ArrayList<>(2);
+ partitionOffset.add(entry.getKey().toString());
+ partitionOffset.add(entry.getValue().toString());
+ propertyValues.add(JavaConversions.asScalaBuffer(partitionOffset).toSeq());
+ }
+ properties.putIndexedFixedProperties(
+ SPECIFIC_OFFSETS,
+ JavaConversions.asScalaBuffer(propertyKeys).toSeq(),
+ JavaConversions.asScalaBuffer(propertyValues).toSeq()
+ );
+ }
+ if (tableJsonMapping.isPresent()) {
+ List<String> propertyKeys = new ArrayList<>();
+ propertyKeys.add(TABLE_FIELD);
+ propertyKeys.add(JSON_FIELD);
+
+ List<Seq<String>> mappingFields = new ArrayList<>(tableJsonMapping.get().size());
+ for (Map.Entry<String, String> entry : tableJsonMapping.get().entrySet()) {
+ List<String> singleMapping = new ArrayList<>(2);
+ singleMapping.add(entry.getKey());
+ singleMapping.add(entry.getValue());
+ mappingFields.add(JavaConversions.asScalaBuffer(singleMapping).toSeq());
+ }
+ properties.putIndexedFixedProperties(
+ TABLE_JSON_MAPPING,
+ JavaConversions.asScalaBuffer(propertyKeys).toSeq(),
+ JavaConversions.asScalaBuffer(mappingFields).toSeq()
+ );
+ }
+ }
+
+ @Override
+ public boolean needsFormat() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
new file mode 100644
index 0000000..a3ca22f
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/KafkaValidator.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.descriptors;
+
+import org.apache.flink.streaming.connectors.kafka.config.StartupMode;
+import org.apache.flink.table.api.ValidationException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import scala.Function0;
+import scala.Tuple2;
+import scala.collection.JavaConversions;
+import scala.runtime.AbstractFunction0;
+import scala.runtime.BoxedUnit;
+
+
+/**
+ * The validator for {@link Kafka}.
+ */
+public class KafkaValidator extends ConnectorDescriptorValidator {
+ // fields
+ public static final String CONNECTOR_TYPE_VALUE = "kafka";
+ public static final String KAFKA_VERSION = "kafka.version";
+ public static final String BOOTSTRAP_SERVERS = "bootstrap.servers";
+ public static final String GROUP_ID = "group.id";
+ public static final String TOPIC = "topic";
+ public static final String STARTUP_MODE = "startup.mode";
+ public static final String SPECIFIC_OFFSETS = "specific.offsets";
+ public static final String TABLE_JSON_MAPPING = "table.json.mapping";
+
+ public static final String PARTITION = "partition";
+ public static final String OFFSET = "offset";
+
+ public static final String TABLE_FIELD = "table.field";
+ public static final String JSON_FIELD = "json.field";
+
+ public static final String ZOOKEEPER_CONNECT = "zookeeper.connect"; // only required for 0.8
+
+ // values
+ public static final String KAFKA_VERSION_VALUE_08 = "0.8";
+ public static final String KAFKA_VERSION_VALUE_09 = "0.9";
+ public static final String KAFKA_VERSION_VALUE_010 = "0.10";
+ public static final String KAFKA_VERSION_VALUE_011 = "0.11";
+
+ public static final String STARTUP_MODE_VALUE_EARLIEST = "earliest-offset";
+ public static final String STARTUP_MODE_VALUE_LATEST = "latest-offset";
+ public static final String STARTUP_MODE_VALUE_GROUP_OFFSETS = "group-offsets";
+ public static final String STARTUP_MODE_VALUE_SPECIFIC_OFFSETS = "specific-offsets";
+
+ // utils
+ public static Map<String, String> normalizeStartupMode(StartupMode startupMode) {
+ Map<String, String> mapPair = new HashMap<>();
+ switch (startupMode) {
+ case EARLIEST:
+ mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_EARLIEST);
+ break;
+ case LATEST:
+ mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_LATEST);
+ break;
+ case GROUP_OFFSETS:
+ mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_GROUP_OFFSETS);
+ break;
+ case SPECIFIC_OFFSETS:
+ mapPair.put(STARTUP_MODE, STARTUP_MODE_VALUE_SPECIFIC_OFFSETS);
+ break;
+ }
+ return mapPair;
+ }
+
+ @Override
+ public void validate(DescriptorProperties properties) {
+ super.validate(properties);
+
+ AbstractFunction0<BoxedUnit> emptyValidator = new AbstractFunction0<BoxedUnit>() {
+ @Override
+ public BoxedUnit apply() {
+ return BoxedUnit.UNIT;
+ }
+ };
+
+ properties.validateValue(CONNECTOR_TYPE(), CONNECTOR_TYPE_VALUE, false);
+
+ AbstractFunction0<BoxedUnit> version08Validator = new AbstractFunction0<BoxedUnit>() {
+ @Override
+ public BoxedUnit apply() {
+ properties.validateString(ZOOKEEPER_CONNECT, false, 0, Integer.MAX_VALUE);
+ return BoxedUnit.UNIT;
+ }
+ };
+
+ Map<String, Function0<BoxedUnit>> versionValidatorMap = new HashMap<>();
+ versionValidatorMap.put(KAFKA_VERSION_VALUE_08, version08Validator);
+ versionValidatorMap.put(KAFKA_VERSION_VALUE_09, emptyValidator);
+ versionValidatorMap.put(KAFKA_VERSION_VALUE_010, emptyValidator);
+ versionValidatorMap.put(KAFKA_VERSION_VALUE_011, emptyValidator);
+ properties.validateEnum(
+ KAFKA_VERSION,
+ false,
+ toScalaImmutableMap(versionValidatorMap)
+ );
+
+ properties.validateString(BOOTSTRAP_SERVERS, false, 1, Integer.MAX_VALUE);
+ properties.validateString(GROUP_ID, false, 1, Integer.MAX_VALUE);
+ properties.validateString(TOPIC, false, 1, Integer.MAX_VALUE);
+
+ AbstractFunction0<BoxedUnit> specificOffsetsValidator = new AbstractFunction0<BoxedUnit>() {
+ @Override
+ public BoxedUnit apply() {
+ Map<String, String> partitions = JavaConversions.mapAsJavaMap(
+ properties.getIndexedProperty(SPECIFIC_OFFSETS, PARTITION));
+
+ Map<String, String> offsets = JavaConversions.mapAsJavaMap(
+ properties.getIndexedProperty(SPECIFIC_OFFSETS, OFFSET));
+ if (partitions.isEmpty() || offsets.isEmpty()) {
+ throw new ValidationException("Offsets must be set for SPECIFIC_OFFSETS mode.");
+ }
+ for (int i = 0; i < partitions.size(); ++i) {
+ properties.validateInt(
+ SPECIFIC_OFFSETS + "." + i + "." + PARTITION,
+ false,
+ 0,
+ Integer.MAX_VALUE);
+ properties.validateLong(
+ SPECIFIC_OFFSETS + "." + i + "." + OFFSET,
+ false,
+ 0,
+ Long.MAX_VALUE);
+ }
+ return BoxedUnit.UNIT;
+ }
+ };
+ Map<String, Function0<BoxedUnit>> startupModeValidatorMap = new HashMap<>();
+ startupModeValidatorMap.put(STARTUP_MODE_VALUE_GROUP_OFFSETS, emptyValidator);
+ startupModeValidatorMap.put(STARTUP_MODE_VALUE_EARLIEST, emptyValidator);
+ startupModeValidatorMap.put(STARTUP_MODE_VALUE_LATEST, emptyValidator);
+ startupModeValidatorMap.put(STARTUP_MODE_VALUE_SPECIFIC_OFFSETS, specificOffsetsValidator);
+
+ properties.validateEnum(STARTUP_MODE, true, toScalaImmutableMap(startupModeValidatorMap));
+ validateTableJsonMapping(properties);
+ }
+
+ private void validateTableJsonMapping(DescriptorProperties properties) {
+ Map<String, String> mappingTableField = JavaConversions.mapAsJavaMap(
+ properties.getIndexedProperty(TABLE_JSON_MAPPING, TABLE_FIELD));
+ Map<String, String> mappingJsonField = JavaConversions.mapAsJavaMap(
+ properties.getIndexedProperty(TABLE_JSON_MAPPING, JSON_FIELD));
+
+ if (mappingJsonField.size() != mappingJsonField.size()) {
+ throw new ValidationException("Table JSON mapping must be one to one.");
+ }
+
+ for (int i = 0; i < mappingTableField.size(); i++) {
+ properties.validateString(
+ TABLE_JSON_MAPPING + "." + i + "." + TABLE_FIELD,
+ false,
+ 1,
+ Integer.MAX_VALUE);
+ properties.validateString(
+ TABLE_JSON_MAPPING + "." + i + "." + JSON_FIELD,
+ false,
+ 1,
+ Integer.MAX_VALUE);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private <K, V> scala.collection.immutable.Map<K, V> toScalaImmutableMap(Map<K, V> javaMap) {
+ final java.util.List<scala.Tuple2<K, V>> list = new java.util.ArrayList<>(javaMap.size());
+ for (final java.util.Map.Entry<K, V> entry : javaMap.entrySet()) {
+ list.add(scala.Tuple2.apply(entry.getKey(), entry.getValue()));
+ }
+ final scala.collection.Seq<Tuple2<K, V>> seq =
+ scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toSeq();
+ return (scala.collection.immutable.Map<K, V>) scala.collection.immutable.Map$.MODULE$.apply(seq);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
new file mode 100644
index 0000000..964a624
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaJsonTableFromDescriptorTestBase.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.api.java.StreamTableEnvironment;
+import org.apache.flink.table.descriptors.Kafka;
+
+import org.mockito.Mockito;
+
+/**
+ * Tests for {@link KafkaJsonTableSourceFactory}.
+ */
+public abstract class KafkaJsonTableFromDescriptorTestBase {
+ private static final String GROUP_ID = "test-group";
+ private static final String BOOTSTRAP_SERVERS = "localhost:1234";
+ private static final String TOPIC = "test-topic";
+
+ protected abstract String versionForTest();
+
+ protected abstract KafkaJsonTableSource.Builder builderForTest();
+
+ protected abstract void extraSettings(KafkaTableSource.Builder builder, Kafka kafka);
+
+ private static StreamExecutionEnvironment env = Mockito.mock(StreamExecutionEnvironment.class);
+ private static StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
+
+// @Test
+// public void buildJsonTableSourceTest() throws Exception {
+// final URL url = getClass().getClassLoader().getResource("kafka-json-schema.json");
+// Objects.requireNonNull(url);
+// final String schema = FileUtils.readFileUtf8(new File(url.getFile()));
+//
+// Map<String, String> tableJsonMapping = new HashMap<>();
+// tableJsonMapping.put("fruit-name", "name");
+// tableJsonMapping.put("fruit-count", "count");
+// tableJsonMapping.put("event-time", "time");
+//
+// // Construct with the builder.
+// Properties props = new Properties();
+// props.put("group.id", GROUP_ID);
+// props.put("bootstrap.servers", BOOTSTRAP_SERVERS);
+//
+// Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
+// specificOffsets.put(new KafkaTopicPartition(TOPIC, 0), 100L);
+// specificOffsets.put(new KafkaTopicPartition(TOPIC, 1), 123L);
+//
+// KafkaTableSource.Builder builder = builderForTest()
+// .forJsonSchema(TableSchema.fromTypeInfo(JsonSchemaConverter.convert(schema)))
+// .failOnMissingField(true)
+// .withTableToJsonMapping(tableJsonMapping)
+// .withKafkaProperties(props)
+// .forTopic(TOPIC)
+// .fromSpecificOffsets(specificOffsets)
+// .withSchema(
+// TableSchema.builder()
+// .field("fruit-name", Types.STRING)
+// .field("fruit-count", Types.INT)
+// .field("event-time", Types.LONG)
+// .field("proc-time", Types.SQL_TIMESTAMP)
+// .build())
+// .withProctimeAttribute("proc-time");
+//
+// // Construct with the descriptor.
+// Map<Integer, Long> offsets = new HashMap<>();
+// offsets.put(0, 100L);
+// offsets.put(1, 123L);
+// Kafka kafka = new Kafka()
+// .version(versionForTest())
+// .groupId(GROUP_ID)
+// .bootstrapServers(BOOTSTRAP_SERVERS)
+// .topic(TOPIC)
+// .startupMode(StartupMode.SPECIFIC_OFFSETS)
+// .specificOffsets(offsets)
+// .tableJsonMapping(tableJsonMapping);
+// extraSettings(builder, kafka);
+//
+// TableSource source = tEnv
+// .from(kafka)
+// .withFormat(
+// new Json()
+// .schema(schema)
+// .failOnMissingField(true))
+// .withSchema(new Schema()
+// .field("fruit-name", Types.STRING)
+// .field("fruit-count", Types.INT)
+// .field("event-time", Types.LONG)
+// .field("proc-time", Types.SQL_TIMESTAMP).proctime())
+// .toTableSource();
+//
+// Assert.assertEquals(builder.build(), source);
+// }
+
+// @Test(expected = TableException.class)
+// public void buildJsonTableSourceFailTest() {
+// tEnv.from(
+// new Kafka()
+// .version(versionForTest())
+// .groupId(GROUP_ID)
+// .bootstrapServers(BOOTSTRAP_SERVERS)
+// .topic(TOPIC)
+// .startupMode(StartupMode.SPECIFIC_OFFSETS)
+// .specificOffsets(new HashMap<>()))
+// .withFormat(
+// new Json()
+// .schema("")
+// .failOnMissingField(true))
+// .toTableSource();
+// }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
new file mode 100644
index 0000000..5167e5e
--- /dev/null
+++ b/flink-connectors/flink-connector-kafka-base/src/test/resources/kafka-json-schema.json
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{
+ "title": "Fruit",
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "count": {
+ "type": "integer"
+ },
+ "time": {
+ "description": "Age in years",
+ "type": "number"
+ }
+ },
+ "required": ["name", "count", "time"]
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1d26062d/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
index 534ef39..1e88d93 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/TableSchema.scala
@@ -92,6 +92,11 @@ class TableSchema(
}
/**
+ * Returns the number of columns.
+ */
+ def getColumnNum: Int = columnNames.length
+
+ /**
* Returns all column names as an array.
*/
def getColumnNames: Array[String] = columnNames
[2/5] flink git commit: [FLINK-8538] [table] Improve unified table
sources
Posted by tw...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
index d112732..555d92d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/DescriptorProperties.scala
@@ -21,24 +21,32 @@ package org.apache.flink.table.descriptors
import java.io.Serializable
import java.lang.{Boolean => JBoolean, Double => JDouble, Integer => JInt, Long => JLong}
import java.util
+import java.util.function.{Consumer, Supplier}
import java.util.regex.Pattern
+import java.util.{Optional, List => JList, Map => JMap}
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang.StringEscapeUtils
import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.table.api.{TableSchema, ValidationException}
-import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema}
+import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
+import org.apache.flink.table.api.{TableException, TableSchema, ValidationException}
+import org.apache.flink.table.descriptors.DescriptorProperties.{NAME, TYPE, normalizeTableSchema, toJava}
import org.apache.flink.table.typeutils.TypeStringUtils
import org.apache.flink.util.InstantiationUtil
import org.apache.flink.util.Preconditions.checkNotNull
-import scala.collection.mutable
import scala.collection.JavaConverters._
+import scala.collection.mutable
/**
* Utility class for having a unified string-based representation of Table API related classes
* such as [[TableSchema]], [[TypeInformation]], etc.
*
+ * '''Note to implementers''': Please try to reuse key names as much as possible. Key-names
+ * should be hierarchical and lower case. Use "-" instead of dots or camel case.
+ * E.g., connector.schema.start-from = from-earliest. Try not to use the higher level in a
+ * key-name. E.g., instead of connector.kafka.kafka-version use connector.kafka.version.
+ *
* @param normalizeKeys flag that indicates if keys should be normalized (this flag is
* necessary for backwards compatibility)
*/
@@ -46,39 +54,18 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
private val properties: mutable.Map[String, String] = new mutable.HashMap[String, String]()
- private def put(key: String, value: String): Unit = {
- if (properties.contains(key)) {
- throw new IllegalStateException("Property already present.")
- }
- if (normalizeKeys) {
- properties.put(key.toLowerCase, value)
- } else {
- properties.put(key, value)
- }
- }
-
- // for testing
- private[flink] def unsafePut(key: String, value: String): Unit = {
- properties.put(key, value)
- }
-
- // for testing
- private[flink] def unsafeRemove(key: String): Unit = {
- properties.remove(key)
- }
-
- def putProperties(properties: Map[String, String]): Unit = {
- properties.foreach { case (k, v) =>
- put(k, v)
- }
- }
-
- def putProperties(properties: util.Map[String, String]): Unit = {
+ /**
+ * Adds a set of properties.
+ */
+ def putProperties(properties: JMap[String, String]): Unit = {
properties.asScala.foreach { case (k, v) =>
put(k, v)
}
}
+ /**
+ * Adds a class under the given key.
+ */
def putClass(key: String, clazz: Class[_]): Unit = {
checkNotNull(key)
checkNotNull(clazz)
@@ -89,43 +76,62 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
put(key, clazz.getName)
}
+ /**
+ * Adds a string under the given key.
+ */
def putString(key: String, str: String): Unit = {
checkNotNull(key)
checkNotNull(str)
put(key, str)
}
+ /**
+ * Adds a boolean under the given key.
+ */
def putBoolean(key: String, b: Boolean): Unit = {
checkNotNull(key)
put(key, b.toString)
}
+ /**
+ * Adds a long under the given key.
+ */
def putLong(key: String, l: Long): Unit = {
checkNotNull(key)
put(key, l.toString)
}
+ /**
+ * Adds an integer under the given key.
+ */
def putInt(key: String, i: Int): Unit = {
checkNotNull(key)
put(key, i.toString)
}
+ /**
+ * Adds a character under the given key.
+ */
def putCharacter(key: String, c: Character): Unit = {
checkNotNull(key)
checkNotNull(c)
put(key, c.toString)
}
+ /**
+ * Adds a table schema under the given key.
+ */
def putTableSchema(key: String, schema: TableSchema): Unit = {
+ checkNotNull(key)
+ checkNotNull(schema)
putTableSchema(key, normalizeTableSchema(schema))
}
- def putTableSchema(key: String, nameAndType: Seq[(String, String)]): Unit = {
- putIndexedFixedProperties(
- key,
- Seq(NAME, TYPE),
- nameAndType.map(t => Seq(t._1, t._2))
- )
+ /**
+ * Adds a table schema under the given key.
+ */
+ def putTableSchema(key: String, nameAndType: JList[JTuple2[String, String]]): Unit = {
+ putTableSchema(key, nameAndType.asScala.map(t => (t.f0, t.f1)))
}
/**
@@ -140,19 +146,12 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
*/
def putIndexedFixedProperties(
key: String,
- propertyKeys: Seq[String],
- propertyValues: Seq[Seq[String]])
+ propertyKeys: JList[String],
+ propertyValues: JList[JList[String]])
: Unit = {
checkNotNull(key)
checkNotNull(propertyValues)
- propertyValues.zipWithIndex.foreach { case (values, idx) =>
- if (values.lengthCompare(propertyKeys.size) != 0) {
- throw new ValidationException("Values must have same arity as keys.")
- }
- values.zipWithIndex.foreach { case (value, keyIdx) =>
- put(s"$key.$idx.${propertyKeys(keyIdx)}", value)
- }
- }
+ putIndexedFixedProperties(key, propertyKeys.asScala, propertyValues.asScala.map(_.asScala))
}
/**
@@ -167,65 +166,163 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
*/
def putIndexedVariableProperties(
key: String,
- propertySets: Seq[Map[String, String]])
+ propertySets: JList[JMap[String, String]])
: Unit = {
checkNotNull(key)
checkNotNull(propertySets)
- propertySets.zipWithIndex.foreach { case (propertySet, idx) =>
- propertySet.foreach { case (k, v) =>
- put(s"$key.$idx.$k", v)
- }
- }
+ putIndexedVariableProperties(key, propertySets.asScala.map(_.asScala.toMap))
}
// ----------------------------------------------------------------------------------------------
- def getString(key: String): Option[String] = {
- properties.get(key)
+ /**
+ * Returns a string value under the given key if it exists.
+ */
+ def getOptionalString(key: String): Optional[String] = toJava(properties.get(key))
+
+ /**
+ * Returns a string value under the given existing key.
+ */
+ def getString(key: String): String = {
+ get(key)
}
- def getCharacter(key: String): Option[Character] = getString(key) match {
- case Some(c) =>
+ /**
+ * Returns a character value under the given key if it exists.
+ */
+ def getOptionalCharacter(key: String): Optional[Character] = {
+ val value = properties.get(key).map { c =>
if (c.length != 1) {
throw new ValidationException(s"The value of $key must only contain one character.")
}
- Some(c.charAt(0))
+ Char.box(c.charAt(0))
+ }
+ toJava(value)
+ }
- case None => None
+ /**
+ * Returns a character value under the given existing key.
+ */
+ def getCharacter(key: String): Char = {
+ getOptionalCharacter(key).orElseThrow(exceptionSupplier(key))
}
- def getBoolean(key: String): Option[Boolean] = getString(key) match {
- case Some(b) => Some(JBoolean.parseBoolean(b))
+ /**
+ * Returns a class value under the given key if it exists.
+ */
+ def getOptionalClass[T](key: String, superClass: Class[T]): Optional[Class[T]] = {
+ val value = properties.get(key).map { name =>
+ val clazz = try {
+ Class.forName(
+ name,
+ true,
+ Thread.currentThread().getContextClassLoader).asInstanceOf[Class[T]]
+ } catch {
+ case e: Exception =>
+ throw new ValidationException(s"Could not get class '$name' for key '$key'.", e)
+ }
+ if (!superClass.isAssignableFrom(clazz)) {
+ throw new ValidationException(s"Class '$name' does not extend from the required " +
+ s"class '${superClass.getName}' for key '$key'.")
+ }
+ clazz
+ }
+ toJava(value)
+ }
- case None => None
+ /**
+ * Returns a class value under the given existing key.
+ */
+ def getClass[T](key: String, superClass: Class[T]): Class[T] = {
+ getOptionalClass(key, superClass).orElseThrow(exceptionSupplier(key))
}
- def getInt(key: String): Option[Int] = getString(key) match {
- case Some(l) => Some(JInt.parseInt(l))
+ /**
+ * Returns a boolean value under the given key if it exists.
+ */
+ def getOptionalBoolean(key: String): Optional[JBoolean] = {
+ val value = properties.get(key).map(JBoolean.parseBoolean(_)).map(Boolean.box)
+ toJava(value)
+ }
+
+ /**
+ * Returns a boolean value under the given existing key.
+ */
+ def getBoolean(key: String): Boolean = {
+ getOptionalBoolean(key).orElseThrow(exceptionSupplier(key))
+ }
- case None => None
+ /**
+ * Returns an integer value under the given key if it exists.
+ */
+ def getOptionalInt(key: String): Optional[JInt] = {
+ val value = properties.get(key).map(JInt.parseInt(_)).map(Int.box)
+ toJava(value)
}
- def getLong(key: String): Option[Long] = getString(key) match {
- case Some(l) => Some(JLong.parseLong(l))
+ /**
+ * Returns an integer value under the given existing key.
+ */
+ def getInt(key: String): Int = {
+ getOptionalInt(key).orElseThrow(exceptionSupplier(key))
+ }
- case None => None
+ /**
+ * Returns a long value under the given key if it exists.
+ */
+ def getOptionalLong(key: String): Optional[JLong] = {
+ val value = properties.get(key).map(JLong.parseLong(_)).map(Long.box)
+ toJava(value)
+ }
+
+ /**
+ * Returns a long value under the given existing key.
+ */
+ def getLong(key: String): Long = {
+ getOptionalLong(key).orElseThrow(exceptionSupplier(key))
+ }
+
+ /**
+ * Returns a double value under the given key if it exists.
+ */
+ def getOptionalDouble(key: String): Optional[JDouble] = {
+ val value = properties.get(key).map(JDouble.parseDouble(_)).map(Double.box)
+ toJava(value)
+ }
+
+ /**
+ * Returns a double value under the given key if it exists.
+ */
+ def getDouble(key: String): Double = {
+ getOptionalDouble(key).orElseThrow(exceptionSupplier(key))
}
- def getDouble(key: String): Option[Double] = getString(key) match {
- case Some(d) => Some(JDouble.parseDouble(d))
+ /**
+ * Returns the type information under the given key if it exists.
+ */
+ def getOptionalType(key: String): Optional[TypeInformation[_]] = {
+ val value = properties.get(key).map(TypeStringUtils.readTypeInfo)
+ toJava(value)
+ }
- case None => None
+ /**
+ * Returns the type information under the given existing key.
+ */
+ def getType(key: String): TypeInformation[_] = {
+ getOptionalType(key).orElseThrow(exceptionSupplier(key))
}
- def getTableSchema(key: String): Option[TableSchema] = {
+ /**
+ * Returns a table schema under the given key if it exists.
+ */
+ def getOptionalTableSchema(key: String): Optional[TableSchema] = {
// filter for number of columns
val fieldCount = properties
.filterKeys(k => k.startsWith(key) && k.endsWith(s".$NAME"))
.size
if (fieldCount == 0) {
- return None
+ return toJava(None)
}
// validate fields and build schema
@@ -243,16 +340,186 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
)
)
}
- Some(schemaBuilder.build())
+ toJava(Some(schemaBuilder.build()))
+ }
+
+ /**
+ * Returns a table schema under the given existing key.
+ */
+ def getTableSchema(key: String): TableSchema = {
+ getOptionalTableSchema(key).orElseThrow(exceptionSupplier(key))
+ }
+
+ /**
+ * Returns the property keys of fixed indexed properties.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ *
+ * getFixedIndexedProperties("schema.fields", List("type", "name")) leads to:
+ *
+ * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
+ * 1: Map("type" -> "schema.fields.1.type", "name" -> "schema.fields.1.name")
+ */
+ def getFixedIndexedProperties(
+ key: String,
+ propertyKeys: JList[String])
+ : JList[JMap[String, String]] = {
+
+ val keys = propertyKeys.asScala
+
+ // filter for index
+ val escapedKey = Pattern.quote(key)
+ val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)")
+
+ // extract index and property keys
+ val indexes = properties.keys.flatMap { k =>
+ val matcher = pattern.matcher(k)
+ if (matcher.find()) {
+ Some(JInt.parseInt(matcher.group(1)))
+ } else {
+ None
+ }
+ }
+
+ // determine max index
+ val maxIndex = indexes.reduceOption(_ max _).getOrElse(-1)
+
+ // validate and create result
+ val list = new util.ArrayList[JMap[String, String]]()
+ for (i <- 0 to maxIndex) {
+ val map = new util.HashMap[String, String]()
+
+ keys.foreach { subKey =>
+ val fullKey = s"$key.$i.$subKey"
+ // check for existence of full key
+ if (!containsKey(fullKey)) {
+ throw exceptionSupplier(fullKey).get()
+ }
+ map.put(subKey, fullKey)
+ }
+
+ list.add(map)
+ }
+ list
+ }
+
+ /**
+ * Returns the property keys of variable indexed properties.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG
+ *
+ * getFixedIndexedProperties("schema.fields", List("type")) leads to:
+ *
+ * 0: Map("type" -> "schema.fields.0.type", "name" -> "schema.fields.0.name")
+ * 1: Map("type" -> "schema.fields.1.type")
+ */
+ def getVariableIndexedProperties(
+ key: String,
+ requiredKeys: JList[String])
+ : JList[JMap[String, String]] = {
+
+ val keys = requiredKeys.asScala
+
+ // filter for index
+ val escapedKey = Pattern.quote(key)
+ val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)")
+
+ // extract index and property keys
+ val indexes = properties.keys.flatMap { k =>
+ val matcher = pattern.matcher(k)
+ if (matcher.find()) {
+ Some((JInt.parseInt(matcher.group(1)), matcher.group(2)))
+ } else {
+ None
+ }
+ }
+
+ // determine max index
+ val maxIndex = indexes.map(_._1).reduceOption(_ max _).getOrElse(-1)
+
+ // validate and create result
+ val list = new util.ArrayList[JMap[String, String]]()
+ for (i <- 0 to maxIndex) {
+ val map = new util.HashMap[String, String]()
+
+ // check and add required keys
+ keys.foreach { subKey =>
+ val fullKey = s"$key.$i.$subKey"
+ // check for existence of full key
+ if (!containsKey(fullKey)) {
+ throw exceptionSupplier(fullKey).get()
+ }
+ map.put(subKey, fullKey)
+ }
+
+ // add optional keys
+ indexes.filter(_._1 == i).foreach { case (_, subKey) =>
+ val fullKey = s"$key.$i.$subKey"
+ map.put(subKey, fullKey)
+ }
+
+ list.add(map)
+ }
+ list
+ }
+
+ /**
+ * Returns all properties under a given key that contains an index in between.
+ *
+ * E.g. rowtime.0.name -> returns all rowtime.#.name properties
+ */
+ def getIndexedProperty(key: String, property: String): JMap[String, String] = {
+ val escapedKey = Pattern.quote(key)
+ properties.filterKeys(k => k.matches(s"$escapedKey\\.\\d+\\.$property")).asJava
+ }
+
+ /**
+ * Returns a prefix subset of properties.
+ */
+ def getPrefix(prefixKey: String): JMap[String, String] = {
+ val prefix = prefixKey + '.'
+ properties.filterKeys(_.startsWith(prefix)).toSeq.map{ case (k, v) =>
+ k.substring(prefix.length) -> v // remove prefix
+ }.toMap.asJava
}
// ----------------------------------------------------------------------------------------------
+ /**
+ * Validates a string property.
+ */
+ def validateString(
+ key: String,
+ isOptional: Boolean)
+ : Unit = {
+ validateString(key, isOptional, 0, Integer.MAX_VALUE)
+ }
+
+ /**
+ * Validates a string property. The boundaries are inclusive.
+ */
def validateString(
key: String,
isOptional: Boolean,
- minLen: Int = 0, // inclusive
- maxLen: Int = Integer.MAX_VALUE) // inclusive
+ minLen: Int) // inclusive
+ : Unit = {
+ validateString(key, isOptional, minLen, Integer.MAX_VALUE)
+ }
+
+ /**
+ * Validates a string property. The boundaries are inclusive.
+ */
+ def validateString(
+ key: String,
+ isOptional: Boolean,
+ minLen: Int, // inclusive
+ maxLen: Int) // inclusive
: Unit = {
if (!properties.contains(key)) {
@@ -269,11 +536,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates an integer property.
+ */
+ def validateInt(
+ key: String,
+ isOptional: Boolean)
+ : Unit = {
+ validateInt(key, isOptional, Int.MinValue, Int.MaxValue)
+ }
+
+ /**
+ * Validates an integer property. The boundaries are inclusive.
+ */
def validateInt(
key: String,
isOptional: Boolean,
- min: Int = Int.MinValue, // inclusive
- max: Int = Int.MaxValue) // inclusive
+ min: Int) // inclusive
+ : Unit = {
+ validateInt(key, isOptional, min, Int.MaxValue)
+ }
+
+ /**
+ * Validates an integer property. The boundaries are inclusive.
+ */
+ def validateInt(
+ key: String,
+ isOptional: Boolean,
+ min: Int, // inclusive
+ max: Int) // inclusive
: Unit = {
if (!properties.contains(key)) {
@@ -295,11 +586,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates a long property.
+ */
+ def validateLong(
+ key: String,
+ isOptional: Boolean)
+ : Unit = {
+ validateLong(key, isOptional, Long.MinValue, Long.MaxValue)
+ }
+
+ /**
+ * Validates a long property. The boundaries are inclusive.
+ */
+ def validateLong(
+ key: String,
+ isOptional: Boolean,
+ min: Long) // inclusive
+ : Unit = {
+ validateLong(key, isOptional, min, Long.MaxValue)
+ }
+
+ /**
+ * Validates a long property. The boundaries are inclusive.
+ */
def validateLong(
key: String,
isOptional: Boolean,
- min: Long = Long.MinValue, // inclusive
- max: Long = Long.MaxValue) // inclusive
+ min: Long, // inclusive
+ max: Long) // inclusive
: Unit = {
if (!properties.contains(key)) {
@@ -321,6 +636,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates that a certain value is present under the given key.
+ */
def validateValue(key: String, value: String, isOptional: Boolean): Unit = {
if (!properties.contains(key)) {
if (!isOptional) {
@@ -334,6 +652,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates that a boolean value is present under the given key.
+ */
def validateBoolean(key: String, isOptional: Boolean): Unit = {
if (!properties.contains(key)) {
if (!isOptional) {
@@ -348,11 +669,35 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates a double property.
+ */
+ def validateDouble(
+ key: String,
+ isOptional: Boolean)
+ : Unit = {
+ validateDouble(key, isOptional, Double.MinValue, Double.MaxValue)
+ }
+
+ /**
+ * Validates a double property. The boundaries are inclusive.
+ */
+ def validateDouble(
+ key: String,
+ isOptional: Boolean,
+ min: Double) // inclusive
+ : Unit = {
+ validateDouble(key, isOptional, min, Double.MaxValue)
+ }
+
+ /**
+ * Validates a double property. The boundaries are inclusive.
+ */
def validateDouble(
key: String,
isOptional: Boolean,
- min: Double = Double.MinValue, // inclusive
- max: Double = Double.MaxValue) // inclusive
+ min: Double, // inclusive
+ max: Double) // inclusive
: Unit = {
if (!properties.contains(key)) {
@@ -374,25 +719,117 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validation for variable indexed properties.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG
+ *
+ * The propertyKeys map defines e.g. "type" and a validation logic for the given full key.
+ *
+ * The validation consumer takes the current prefix e.g. "schema.fields.1.".
+ */
+ def validateVariableIndexedProperties(
+ key: String,
+ allowEmpty: Boolean,
+ propertyKeys: JMap[String, Consumer[String]],
+ requiredKeys: JList[String])
+ : Unit = {
+
+ val keys = propertyKeys.asScala
+
+ // filter for index
+ val escapedKey = Pattern.quote(key)
+ val pattern = Pattern.compile(s"$escapedKey\\.(\\d+)\\.(.*)")
+
+ // extract index and property keys
+ val indexes = properties.keys.flatMap { k =>
+ val matcher = pattern.matcher(k)
+ if (matcher.find()) {
+ Some(JInt.parseInt(matcher.group(1)))
+ } else {
+ None
+ }
+ }
+
+ // determine max index
+ val maxIndex = indexes.reduceOption(_ max _).getOrElse(-1)
+
+ if (maxIndex < 0 && !allowEmpty) {
+ throw new ValidationException(s"Property key '$key' must not be empty.")
+ }
+
+ // validate
+ for (i <- 0 to maxIndex) {
+ keys.foreach { case (subKey, validation) =>
+ val fullKey = s"$key.$i.$subKey"
+ // only validate if it exists
+ if (properties.contains(fullKey)) {
+ validation.accept(s"$key.$i.")
+ } else {
+ // check if it is required
+ if (requiredKeys.contains(subKey)) {
+ throw new ValidationException(s"Required property key '$fullKey' is missing.")
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Validation for fixed indexed properties.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ *
+ * The propertyKeys map must define e.g. "type" and "name" and a validation logic for the
+ * given full key.
+ */
+ def validateFixedIndexedProperties(
+ key: String,
+ allowEmpty: Boolean,
+ propertyKeys: JMap[String, Consumer[String]])
+ : Unit = {
+
+ validateVariableIndexedProperties(
+ key,
+ allowEmpty,
+ propertyKeys,
+ new util.ArrayList(propertyKeys.keySet()))
+ }
+
+ /**
+ * Validates a table schema property.
+ */
def validateTableSchema(key: String, isOptional: Boolean): Unit = {
- // filter for name columns
- val names = getIndexedProperty(key, NAME)
- // filter for type columns
- val types = getIndexedProperty(key, TYPE)
- if (names.isEmpty && types.isEmpty && !isOptional) {
- throw new ValidationException(
- s"Could not find the required schema for property '$key'.")
+ val nameValidation = (prefix: String) => {
+ validateString(prefix + NAME, isOptional = false, minLen = 1)
}
- for (i <- 0 until Math.max(names.size, types.size)) {
- validateString(s"$key.$i.$NAME", isOptional = false, minLen = 1)
- validateType(s"$key.$i.$TYPE", isOptional = false)
+ val typeValidation = (prefix: String) => {
+ validateType(prefix + TYPE, isOptional = false)
}
+
+ validateFixedIndexedProperties(
+ key,
+ isOptional,
+ Map(
+ NAME -> toJava(nameValidation),
+ TYPE -> toJava(typeValidation)
+ ).asJava
+ )
}
+ /**
+ * Validates a enum property with a set of validation logic for each enum value.
+ */
def validateEnum(
key: String,
isOptional: Boolean,
- enumToValidation: Map[String, () => Unit])
+ enumToValidation: JMap[String, Consumer[String]])
: Unit = {
if (!properties.contains(key)) {
@@ -401,15 +838,26 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
} else {
val value = properties(key)
- if (!enumToValidation.contains(value)) {
+ if (!enumToValidation.containsKey(value)) {
throw new ValidationException(s"Unknown value for property '$key'. " +
- s"Supported values [${enumToValidation.keys.mkString(", ")}] but was: $value")
+ s"Supported values [${enumToValidation.keySet().asScala.mkString(", ")}] but was: $value")
} else {
- enumToValidation(value).apply() // run validation logic
+ // run validation logic
+ enumToValidation.get(value).accept(key)
}
}
}
+ /**
+ * Validates a enum property with a set of enum values.
+ */
+ def validateEnumValues(key: String, isOptional: Boolean, values: JList[String]): Unit = {
+ validateEnum(key, isOptional, values.asScala.map((_, noValidation())).toMap.asJava)
+ }
+
+ /**
+ * Validates a type property.
+ */
def validateType(key: String, isOptional: Boolean): Unit = {
if (!properties.contains(key)) {
if (!isOptional) {
@@ -420,6 +868,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates that the given prefix is not included in these properties.
+ */
def validatePrefixExclusion(prefix: String): Unit = {
val invalidField = properties.find(_._1.startsWith(prefix))
if (invalidField.isDefined) {
@@ -428,6 +879,9 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
}
}
+ /**
+ * Validates that the given key is not included in these properties.
+ */
def validateExclusion(key: String): Unit = {
if (properties.contains(key)) {
throw new ValidationException(s"Property '$key' is not allowed in this context.")
@@ -436,28 +890,159 @@ class DescriptorProperties(normalizeKeys: Boolean = true) {
// ----------------------------------------------------------------------------------------------
- def getIndexedProperty(key: String, property: String): Map[String, String] = {
- val escapedKey = Pattern.quote(key)
- properties.filterKeys(k => k.matches(s"$escapedKey\\.\\d+\\.$property")).toMap
+ /**
+ * Returns if any property contains parts of a given string.
+ */
+ def containsString(str: String): Boolean = {
+ properties.exists(e => e._1.contains(str))
}
- def contains(str: String): Boolean = {
- properties.exists(e => e._1.contains(str))
+ /**
+ * Returns if the given key is contained.
+ */
+ def containsKey(key: String): Boolean = {
+ properties.contains(key)
}
+ /**
+ * Returns if a given prefix exists in the properties.
+ */
def hasPrefix(prefix: String): Boolean = {
properties.exists(e => e._1.startsWith(prefix))
}
- def asMap: Map[String, String] = {
- properties.toMap
+ /**
+ * Returns a Scala Map.
+ */
+ def asMap: JMap[String, String] = {
+ properties.toMap.asJava
+ }
+
+ // ----------------------------------------------------------------------------------------------
+
+ /**
+ * Returns an empty validation logic.
+ */
+ def noValidation(): Consumer[String] = DescriptorProperties.emptyConsumer
+
+ def exceptionSupplier(key: String): Supplier[TableException] = new Supplier[TableException] {
+ override def get(): TableException = {
+ new TableException(s"Property with key '$key' could not be found. " +
+ s"This is a bug because the validation logic should have checked that before.")
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------
+
+ /**
+ * Adds a property.
+ */
+ private def put(key: String, value: String): Unit = {
+ if (properties.contains(key)) {
+ throw new IllegalStateException("Property already present.")
+ }
+ if (normalizeKeys) {
+ properties.put(key.toLowerCase, value)
+ } else {
+ properties.put(key, value)
+ }
+ }
+
+ /**
+ * Gets an existing property.
+ */
+ private def get(key: String): String = {
+ properties.getOrElse(
+ key,
+ throw exceptionSupplier(key).get())
+ }
+
+ /**
+ * Raw access to the underlying properties map for testing purposes.
+ */
+ private[flink] def unsafePut(key: String, value: String): Unit = {
+ properties.put(key, value)
+ }
+
+ /**
+ * Raw access to the underlying properties map for testing purposes.
+ */
+ private[flink] def unsafeRemove(key: String): Unit = {
+ properties.remove(key)
+ }
+
+ /**
+ * Adds a table schema under the given key.
+ */
+ private def putTableSchema(key: String, nameAndType: Seq[(String, String)]): Unit = {
+ putIndexedFixedProperties(
+ key,
+ Seq(NAME, TYPE),
+ nameAndType.map(t => Seq(t._1, t._2))
+ )
+ }
+
+ /**
+ * Adds an indexed sequence of properties (with sub-properties) under a common key.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.type = LONG, schema.fields.1.name = test2
+ *
+ * The arity of each propertyValue must match the arity of propertyKeys.
+ */
+ private def putIndexedFixedProperties(
+ key: String,
+ propertyKeys: Seq[String],
+ propertyValues: Seq[Seq[String]])
+ : Unit = {
+ checkNotNull(key)
+ checkNotNull(propertyValues)
+ propertyValues.zipWithIndex.foreach { case (values, idx) =>
+ if (values.lengthCompare(propertyKeys.size) != 0) {
+ throw new ValidationException("Values must have same arity as keys.")
+ }
+ values.zipWithIndex.foreach { case (value, keyIdx) =>
+ put(s"$key.$idx.${propertyKeys(keyIdx)}", value)
+ }
+ }
+ }
+
+ /**
+ * Adds an indexed mapping of properties under a common key.
+ *
+ * For example:
+ *
+ * schema.fields.0.type = INT, schema.fields.0.name = test
+ * schema.fields.1.name = test2
+ *
+ * The arity of the propertySets can differ.
+ */
+ private def putIndexedVariableProperties(
+ key: String,
+ propertySets: Seq[Map[String, String]])
+ : Unit = {
+ checkNotNull(key)
+ checkNotNull(propertySets)
+ propertySets.zipWithIndex.foreach { case (propertySet, idx) =>
+ propertySet.foreach { case (k, v) =>
+ put(s"$key.$idx.$k", v)
+ }
+ }
}
}
object DescriptorProperties {
- val TYPE = "type"
- val NAME = "name"
+ private val emptyConsumer: Consumer[String] = new Consumer[String] {
+ override def accept(t: String): Unit = {
+ // nothing to do
+ }
+ }
+
+ val TYPE: String = "type"
+ val NAME: String = "name"
// the string representation should be equal to SqlTypeName
def normalizeTypeInfo(typeInfo: TypeInformation[_]): String = {
@@ -487,6 +1072,24 @@ object DescriptorProperties {
}
}
+ def deserialize[T](data: String, expected: Class[T]): T = {
+ try {
+ val byteData = Base64.decodeBase64(data)
+ val obj = InstantiationUtil.deserializeObject[T](
+ byteData,
+ Thread.currentThread.getContextClassLoader)
+ if (!expected.isAssignableFrom(obj.getClass)) {
+ throw new ValidationException(
+ s"Serialized data contains an object of unexpected type. " +
+ s"Expected '${expected.getName}' but was '${obj.getClass.getName}'")
+ }
+ obj
+ } catch {
+ case e: Exception =>
+ throw new ValidationException(s"Could not deserialize data: '$data'", e)
+ }
+ }
+
def toString(keyOrValue: String): String = {
StringEscapeUtils.escapeJava(keyOrValue)
}
@@ -494,4 +1097,24 @@ object DescriptorProperties {
def toString(key: String, value: String): String = {
toString(key) + "=" + toString(value)
}
+
+ // the following methods help for Scala <-> Java interfaces
+ // most of these methods are not necessary once we upgraded to Scala 2.12
+
+ def toJava[T](option: Option[T]): Optional[T] = option match {
+ case Some(v) => Optional.of(v)
+ case None => Optional.empty()
+ }
+
+ def toScala[T](option: Optional[T]): Option[T] = Option(option.orElse(null.asInstanceOf[T]))
+
+ def toJava[T](func: Function[T, Unit]): Consumer[T] = new Consumer[T] {
+ override def accept(t: T): Unit = {
+ func.apply(t)
+ }
+ }
+
+ def toJava[T0, T1](tuple: (T0, T1)): JTuple2[T0, T1] = {
+ new JTuple2[T0, T1](tuple._1, tuple._2)
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
index b1d900f..f306b5a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FileSystem.scala
@@ -23,7 +23,8 @@ import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, C
/**
* Connector descriptor for a file system.
*/
-class FileSystem extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1) {
+class FileSystem extends ConnectorDescriptor(
+ CONNECTOR_TYPE_VALUE, version = 1, formatNeeded = true) {
private var path: Option[String] = None
@@ -43,8 +44,6 @@ class FileSystem extends ConnectorDescriptor(CONNECTOR_TYPE_VALUE, version = 1)
override protected def addConnectorProperties(properties: DescriptorProperties): Unit = {
path.foreach(properties.putString(CONNECTOR_PATH, _))
}
-
- override private[flink] def needsFormat() = true
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
index 86f6229..bca67c6 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptor.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION}
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_PROPERTY_VERSION}
/**
* Describes the format of data.
@@ -37,7 +37,7 @@ abstract class FormatDescriptor(
*/
final private[flink] def addProperties(properties: DescriptorProperties): Unit = {
properties.putString(FORMAT_TYPE, tpe)
- properties.putInt(FORMAT_VERSION, version)
+ properties.putInt(FORMAT_PROPERTY_VERSION, version)
addFormatProperties(properties)
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
index 1aaa399..301189a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/FormatDescriptorValidator.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION}
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE}
/**
* Validator for [[FormatDescriptor]].
@@ -27,13 +27,32 @@ class FormatDescriptorValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
properties.validateString(FORMAT_TYPE, isOptional = false, minLen = 1)
- properties.validateInt(FORMAT_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateInt(FORMAT_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
}
}
object FormatDescriptorValidator {
+ /**
+ * Key for describing the type of the format. Usually used for factory discovery.
+ */
val FORMAT_TYPE = "format.type"
+
+ /**
+ * Key for describing the property version. This property can be used for backwards
+ * compatibility in case the property format changes.
+ */
+ val FORMAT_PROPERTY_VERSION = "format.property-version"
+
+ /**
+ * Key for describing the version of the format. This property can be used for different
+ * format versions (e.g. Avro 1.8.2 or Avro 2.0).
+ */
val FORMAT_VERSION = "format.version"
+ /**
+ * Key for deriving the schema of the format from the table's schema.
+ */
+ val FORMAT_DERIVE_SCHEMA = "format.derive-schema"
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala
deleted file mode 100644
index cc46d9c..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Json.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.JsonValidator.{FORMAT_FAIL_ON_MISSING_FIELD, FORMAT_SCHEMA_STRING, FORMAT_TYPE_VALUE}
-
-/**
- * Encoding descriptor for JSON.
- */
-class Json extends FormatDescriptor(FORMAT_TYPE_VALUE, version = 1) {
-
- private var failOnMissingField: Option[Boolean] = None
-
- private var schema: Option[String] = None
-
- /**
- * Sets flag whether to fail if a field is missing or not.
- *
- * @param failOnMissingField If set to true, the operation fails if there is a missing field.
- * If set to false, a missing field is set to null.
- * @return The builder.
- */
- def failOnMissingField(failOnMissingField: Boolean): Json = {
- this.failOnMissingField = Some(failOnMissingField)
- this
- }
-
- /**
- * Sets the JSON schema string with field names and the types according to the JSON schema
- * specification [[http://json-schema.org/specification.html]]. Required.
- *
- * The schema might be nested.
- *
- * @param schema JSON schema
- */
- def schema(schema: String): Json = {
- this.schema = Some(schema)
- this
- }
-
- /**
- * Internal method for format properties conversion.
- */
- override protected def addFormatProperties(properties: DescriptorProperties): Unit = {
- // we distinguish between "schema string" and "schema" to allow parsing of a
- // schema object in the future (such that the entire JSON schema can be defined in a YAML
- // file instead of one large string)
- schema.foreach(properties.putString(FORMAT_SCHEMA_STRING, _))
- failOnMissingField.foreach(properties.putBoolean(FORMAT_FAIL_ON_MISSING_FIELD, _))
- }
-}
-
-/**
- * Encoding descriptor for JSON.
- */
-object Json {
-
- /**
- * Encoding descriptor for JSON.
- */
- def apply(): Json = new Json()
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala
deleted file mode 100644
index 9f11caf..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/JsonValidator.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.descriptors
-
-import org.apache.flink.table.descriptors.JsonValidator.{FORMAT_FAIL_ON_MISSING_FIELD, FORMAT_SCHEMA_STRING}
-
-/**
- * Validator for [[Json]].
- */
-class JsonValidator extends FormatDescriptorValidator {
-
- override def validate(properties: DescriptorProperties): Unit = {
- super.validate(properties)
- properties.validateString(FORMAT_SCHEMA_STRING, isOptional = false, minLen = 1)
- properties.validateBoolean(FORMAT_FAIL_ON_MISSING_FIELD, isOptional = true)
- }
-}
-
-object JsonValidator {
-
- val FORMAT_TYPE_VALUE = "json"
- val FORMAT_SCHEMA_STRING = "format.schema-string"
- val FORMAT_FAIL_ON_MISSING_FIELD = "format.fail-on-missing-field"
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
index a8d580c..6631e22 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/MetadataValidator.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME, METADATA_VERSION}
+import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, METADATA_CREATION_TIME, METADATA_LAST_ACCESS_TIME, METADATA_PROPERTY_VERSION}
/**
* Validator for [[Metadata]].
@@ -26,7 +26,7 @@ import org.apache.flink.table.descriptors.MetadataValidator.{METADATA_COMMENT, M
class MetadataValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(METADATA_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateInt(METADATA_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
properties.validateString(METADATA_COMMENT, isOptional = true)
properties.validateLong(METADATA_CREATION_TIME, isOptional = true)
properties.validateLong(METADATA_LAST_ACCESS_TIME, isOptional = true)
@@ -35,7 +35,7 @@ class MetadataValidator extends DescriptorValidator {
object MetadataValidator {
- val METADATA_VERSION = "metadata.version"
+ val METADATA_PROPERTY_VERSION = "metadata.property-version"
val METADATA_COMMENT = "metadata.comment"
val METADATA_CREATION_TIME = "metadata.creation-time"
val METADATA_LAST_ACCESS_TIME = "metadata.last-access-time"
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
index a1c80f5..ed3854d 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Rowtime.scala
@@ -19,11 +19,12 @@
package org.apache.flink.table.descriptors
import org.apache.flink.table.api.Types
-import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_VERSION, normalizeTimestampExtractor, normalizeWatermarkStrategy}
+import org.apache.flink.table.descriptors.RowtimeValidator.{normalizeTimestampExtractor, normalizeWatermarkStrategy}
import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor}
import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy}
import scala.collection.mutable
+import scala.collection.JavaConverters._
/**
* Rowtime descriptor for describing an event time attribute in the schema.
@@ -111,12 +112,9 @@ class Rowtime extends Descriptor {
*/
final override def addProperties(properties: DescriptorProperties): Unit = {
val props = mutable.HashMap[String, String]()
- props.put(ROWTIME_VERSION, "1")
timestampExtractor.foreach(normalizeTimestampExtractor(_).foreach(e => props.put(e._1, e._2)))
watermarkStrategy.foreach(normalizeWatermarkStrategy(_).foreach(e => props.put(e._1, e._2)))
-
- // use a list for the rowtime to support multiple rowtime attributes in the future
- properties.putIndexedVariableProperties(ROWTIME, Seq(props.toMap))
+ properties.putProperties(props.toMap.asJava)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
index 74e49f1..fdec820 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/RowtimeValidator.scala
@@ -18,58 +18,62 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.descriptors.DescriptorProperties.serialize
+import org.apache.flink.table.descriptors.DescriptorProperties.{serialize, toJava}
import org.apache.flink.table.descriptors.RowtimeValidator._
import org.apache.flink.table.sources.tsextractors.{ExistingField, StreamRecordTimestamp, TimestampExtractor}
import org.apache.flink.table.sources.wmstrategies.{AscendingTimestamps, BoundedOutOfOrderTimestamps, PreserveWatermarks, WatermarkStrategy}
+import scala.collection.JavaConverters._
+
/**
* Validator for [[Rowtime]].
*/
class RowtimeValidator(val prefix: String = "") extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(prefix + ROWTIME_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
-
- val noValidation = () => {}
-
- val timestampExistingField = () => {
- properties.validateString(prefix + TIMESTAMPS_FROM, isOptional = false, minLen = 1)
+ val timestampExistingField = (_: String) => {
+ properties.validateString(
+ prefix + ROWTIME_TIMESTAMPS_FROM, isOptional = false, minLen = 1)
}
- val timestampCustom = () => {
- properties.validateString(prefix + TIMESTAMPS_CLASS, isOptional = false, minLen = 1)
- properties.validateString(prefix + TIMESTAMPS_SERIALIZED, isOptional = false, minLen = 1)
+ val timestampCustom = (_: String) => {
+ properties.validateString(
+ prefix + ROWTIME_TIMESTAMPS_CLASS, isOptional = false, minLen = 1)
+ properties.validateString(
+ prefix + ROWTIME_TIMESTAMPS_SERIALIZED, isOptional = false, minLen = 1)
}
properties.validateEnum(
- prefix + TIMESTAMPS_TYPE,
+ prefix + ROWTIME_TIMESTAMPS_TYPE,
isOptional = false,
Map(
- TIMESTAMPS_TYPE_VALUE_FROM_FIELD -> timestampExistingField,
- TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> noValidation,
- TIMESTAMPS_TYPE_VALUE_CUSTOM -> timestampCustom
- )
+ ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD -> toJava(timestampExistingField),
+ ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(),
+ ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM -> toJava(timestampCustom)
+ ).asJava
)
- val watermarkPeriodicBounding = () => {
- properties.validateLong(prefix + WATERMARKS_DELAY, isOptional = false, min = 0)
+ val watermarkPeriodicBounded = (_: String) => {
+ properties.validateLong(
+ prefix + ROWTIME_WATERMARKS_DELAY, isOptional = false, min = 0)
}
- val watermarkCustom = () => {
- properties.validateString(prefix + WATERMARKS_CLASS, isOptional = false, minLen = 1)
- properties.validateString(prefix + WATERMARKS_SERIALIZED, isOptional = false, minLen = 1)
+ val watermarkCustom = (_: String) => {
+ properties.validateString(
+ prefix + ROWTIME_WATERMARKS_CLASS, isOptional = false, minLen = 1)
+ properties.validateString(
+ prefix + ROWTIME_WATERMARKS_SERIALIZED, isOptional = false, minLen = 1)
}
properties.validateEnum(
- prefix + WATERMARKS_TYPE,
+ prefix + ROWTIME_WATERMARKS_TYPE,
isOptional = false,
Map(
- WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> noValidation,
- WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING -> watermarkPeriodicBounding,
- WATERMARKS_TYPE_VALUE_FROM_SOURCE -> noValidation,
- WATERMARKS_TYPE_VALUE_CUSTOM -> watermarkCustom
- )
+ ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING -> properties.noValidation(),
+ ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED -> toJava(watermarkPeriodicBounded),
+ ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE -> properties.noValidation(),
+ ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM -> toJava(watermarkCustom)
+ ).asJava
)
}
}
@@ -77,58 +81,113 @@ class RowtimeValidator(val prefix: String = "") extends DescriptorValidator {
object RowtimeValidator {
val ROWTIME = "rowtime"
-
- // per rowtime properties
-
- val ROWTIME_VERSION = "version"
- val TIMESTAMPS_TYPE = "timestamps.type"
- val TIMESTAMPS_TYPE_VALUE_FROM_FIELD = "from-field"
- val TIMESTAMPS_TYPE_VALUE_FROM_SOURCE = "from-source"
- val TIMESTAMPS_TYPE_VALUE_CUSTOM = "custom"
- val TIMESTAMPS_FROM = "timestamps.from"
- val TIMESTAMPS_CLASS = "timestamps.class"
- val TIMESTAMPS_SERIALIZED = "timestamps.serialized"
-
- val WATERMARKS_TYPE = "watermarks.type"
- val WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING = "periodic-ascending"
- val WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING = "periodic-bounding"
- val WATERMARKS_TYPE_VALUE_FROM_SOURCE = "from-source"
- val WATERMARKS_TYPE_VALUE_CUSTOM = "custom"
- val WATERMARKS_CLASS = "watermarks.class"
- val WATERMARKS_SERIALIZED = "watermarks.serialized"
- val WATERMARKS_DELAY = "watermarks.delay"
+ val ROWTIME_TIMESTAMPS_TYPE = "rowtime.timestamps.type"
+ val ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD = "from-field"
+ val ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE = "from-source"
+ val ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM = "custom"
+ val ROWTIME_TIMESTAMPS_FROM = "rowtime.timestamps.from"
+ val ROWTIME_TIMESTAMPS_CLASS = "rowtime.timestamps.class"
+ val ROWTIME_TIMESTAMPS_SERIALIZED = "rowtime.timestamps.serialized"
+
+ val ROWTIME_WATERMARKS_TYPE = "rowtime.watermarks.type"
+ val ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING = "periodic-ascending"
+ val ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED = "periodic-bounded"
+ val ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE = "from-source"
+ val ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM = "custom"
+ val ROWTIME_WATERMARKS_CLASS = "rowtime.watermarks.class"
+ val ROWTIME_WATERMARKS_SERIALIZED = "rowtime.watermarks.serialized"
+ val ROWTIME_WATERMARKS_DELAY = "rowtime.watermarks.delay"
// utilities
def normalizeTimestampExtractor(extractor: TimestampExtractor): Map[String, String] =
extractor match {
+
case existing: ExistingField =>
Map(
- TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_FROM_FIELD,
- TIMESTAMPS_FROM -> existing.getArgumentFields.apply(0))
+ ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD,
+ ROWTIME_TIMESTAMPS_FROM -> existing.getArgumentFields.apply(0))
+
case _: StreamRecordTimestamp =>
- Map(TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_FROM_SOURCE)
+ Map(ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE)
+
case _: TimestampExtractor =>
Map(
- TIMESTAMPS_TYPE -> TIMESTAMPS_TYPE_VALUE_CUSTOM,
- TIMESTAMPS_CLASS -> extractor.getClass.getName,
- TIMESTAMPS_SERIALIZED -> serialize(extractor))
+ ROWTIME_TIMESTAMPS_TYPE -> ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM,
+ ROWTIME_TIMESTAMPS_CLASS -> extractor.getClass.getName,
+ ROWTIME_TIMESTAMPS_SERIALIZED -> serialize(extractor))
}
def normalizeWatermarkStrategy(strategy: WatermarkStrategy): Map[String, String] =
strategy match {
+
case _: AscendingTimestamps =>
- Map(WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING)
- case bounding: BoundedOutOfOrderTimestamps =>
+ Map(ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING)
+
+ case bounded: BoundedOutOfOrderTimestamps =>
Map(
- WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDING,
- WATERMARKS_DELAY -> bounding.delay.toString)
+ ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED,
+ ROWTIME_WATERMARKS_DELAY -> bounded.delay.toString)
+
case _: PreserveWatermarks =>
- Map(WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_FROM_SOURCE)
+ Map(ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE)
+
case _: WatermarkStrategy =>
Map(
- WATERMARKS_TYPE -> WATERMARKS_TYPE_VALUE_CUSTOM,
- WATERMARKS_CLASS -> strategy.getClass.getName,
- WATERMARKS_SERIALIZED -> serialize(strategy))
+ ROWTIME_WATERMARKS_TYPE -> ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM,
+ ROWTIME_WATERMARKS_CLASS -> strategy.getClass.getName,
+ ROWTIME_WATERMARKS_SERIALIZED -> serialize(strategy))
}
+
+ def getRowtimeComponents(properties: DescriptorProperties, prefix: String)
+ : Option[(TimestampExtractor, WatermarkStrategy)] = {
+
+ // create timestamp extractor
+ val t = properties.getOptionalString(prefix + ROWTIME_TIMESTAMPS_TYPE)
+ if (!t.isPresent) {
+ return None
+ }
+ val extractor: TimestampExtractor = t.get() match {
+
+ case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD =>
+ val field = properties.getString(prefix + ROWTIME_TIMESTAMPS_FROM)
+ new ExistingField(field)
+
+ case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE =>
+ new StreamRecordTimestamp
+
+ case ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM =>
+ val clazz = properties.getClass(
+ ROWTIME_TIMESTAMPS_CLASS,
+ classOf[TimestampExtractor])
+ DescriptorProperties.deserialize(
+ properties.getString(prefix + ROWTIME_TIMESTAMPS_SERIALIZED),
+ clazz)
+ }
+
+ // create watermark strategy
+ val s = properties.getString(prefix + ROWTIME_WATERMARKS_TYPE)
+ val strategy: WatermarkStrategy = s match {
+
+ case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING =>
+ new AscendingTimestamps()
+
+ case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED =>
+ val delay = properties.getLong(prefix + ROWTIME_WATERMARKS_DELAY)
+ new BoundedOutOfOrderTimestamps(delay)
+
+ case ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE =>
+ PreserveWatermarks.INSTANCE
+
+ case ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM =>
+ val clazz = properties.getClass(
+ prefix + ROWTIME_WATERMARKS_CLASS,
+ classOf[WatermarkStrategy])
+ DescriptorProperties.deserialize(
+ properties.getString(prefix + ROWTIME_WATERMARKS_SERIALIZED),
+ clazz)
+ }
+
+ Some((extractor, strategy))
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
index 2f3a389..fcbb2c7 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Schema.scala
@@ -24,6 +24,7 @@ import org.apache.flink.table.descriptors.DescriptorProperties.{normalizeTableSc
import org.apache.flink.table.descriptors.SchemaValidator._
import scala.collection.mutable
+import scala.collection.JavaConverters._
/**
* Describes a schema of a table.
@@ -80,7 +81,7 @@ class Schema extends Descriptor {
}
val fieldProperties = mutable.LinkedHashMap[String, String]()
- fieldProperties += (TYPE -> fieldType)
+ fieldProperties += (SCHEMA_TYPE -> fieldType)
tableSchema += (fieldName -> fieldProperties)
@@ -100,7 +101,7 @@ class Schema extends Descriptor {
lastField match {
case None => throw new ValidationException("No field previously defined. Use field() before.")
case Some(f) =>
- tableSchema(f) += (FROM -> originFieldName)
+ tableSchema(f) += (SCHEMA_FROM -> originFieldName)
lastField = None
}
this
@@ -115,7 +116,7 @@ class Schema extends Descriptor {
lastField match {
case None => throw new ValidationException("No field defined previously. Use field() before.")
case Some(f) =>
- tableSchema(f) += (PROCTIME -> PROCTIME_VALUE_TRUE)
+ tableSchema(f) += (SCHEMA_PROCTIME -> "true")
lastField = None
}
this
@@ -132,7 +133,7 @@ class Schema extends Descriptor {
case Some(f) =>
val fieldProperties = new DescriptorProperties()
rowtime.addProperties(fieldProperties)
- tableSchema(f) ++= fieldProperties.asMap
+ tableSchema(f) ++= fieldProperties.asMap.asScala
lastField = None
}
this
@@ -142,12 +143,11 @@ class Schema extends Descriptor {
* Internal method for properties conversion.
*/
final override private[flink] def addProperties(properties: DescriptorProperties): Unit = {
- properties.putInt(SCHEMA_VERSION, 1)
properties.putIndexedVariableProperties(
SCHEMA,
tableSchema.toSeq.map { case (name, props) =>
- Map(NAME -> name) ++ props
- }
+ (Map(SCHEMA_NAME -> name) ++ props).asJava
+ }.asJava
)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
index 19c0e41..0a23911 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/SchemaValidator.scala
@@ -18,9 +18,17 @@
package org.apache.flink.table.descriptors
-import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME
+import java.util
+import java.util.Optional
+
+import org.apache.flink.table.api.{TableSchema, ValidationException}
+import org.apache.flink.table.descriptors.DescriptorProperties.{toJava, toScala}
+import org.apache.flink.table.descriptors.RowtimeValidator.{ROWTIME, ROWTIME_TIMESTAMPS_TYPE}
import org.apache.flink.table.descriptors.SchemaValidator._
+import org.apache.flink.table.sources.RowtimeAttributeDescriptor
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
/**
* Validator for [[Schema]].
@@ -28,29 +36,39 @@ import org.apache.flink.table.descriptors.SchemaValidator._
class SchemaValidator(isStreamEnvironment: Boolean = true) extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(SCHEMA_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
-
- val names = properties.getIndexedProperty(SCHEMA, NAME)
- val types = properties.getIndexedProperty(SCHEMA, TYPE)
+ val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME)
+ val types = properties.getIndexedProperty(SCHEMA, SCHEMA_TYPE)
if (names.isEmpty && types.isEmpty) {
- throw new ValidationException(s"Could not find the required schema for property '$SCHEMA'.")
+ throw new ValidationException(
+ s"Could not find the required schema in property '$SCHEMA'.")
}
+ var proctimeFound = false
+
for (i <- 0 until Math.max(names.size, types.size)) {
- properties.validateString(s"$SCHEMA.$i.$NAME", isOptional = false, minLen = 1)
- properties.validateType(s"$SCHEMA.$i.$TYPE", isOptional = false)
- properties.validateString(s"$SCHEMA.$i.$FROM", isOptional = true, minLen = 1)
+ properties
+ .validateString(s"$SCHEMA.$i.$SCHEMA_NAME", isOptional = false, minLen = 1)
+ properties
+ .validateType(s"$SCHEMA.$i.$SCHEMA_TYPE", isOptional = false)
+ properties
+ .validateString(s"$SCHEMA.$i.$SCHEMA_FROM", isOptional = true, minLen = 1)
// either proctime or rowtime
- val proctime = s"$SCHEMA.$i.$PROCTIME"
+ val proctime = s"$SCHEMA.$i.$SCHEMA_PROCTIME"
val rowtime = s"$SCHEMA.$i.$ROWTIME"
- if (properties.contains(proctime)) {
+ if (properties.containsKey(proctime)) {
+ // check the environment
if (!isStreamEnvironment) {
throw new ValidationException(
s"Property '$proctime' is not allowed in a batch environment.")
}
+ // check for only one proctime attribute
+ else if (proctimeFound) {
+ throw new ValidationException("A proctime attribute must only be defined once.")
+ }
// check proctime
properties.validateBoolean(proctime, isOptional = false)
+ proctimeFound = properties.getBoolean(proctime)
// no rowtime
properties.validatePrefixExclusion(rowtime)
} else if (properties.hasPrefix(rowtime)) {
@@ -67,14 +85,129 @@ class SchemaValidator(isStreamEnvironment: Boolean = true) extends DescriptorVal
object SchemaValidator {
val SCHEMA = "schema"
- val SCHEMA_VERSION = "schema.version"
+ val SCHEMA_NAME = "name"
+ val SCHEMA_TYPE = "type"
+ val SCHEMA_PROCTIME = "proctime"
+ val SCHEMA_FROM = "from"
+
+ // utilities
+
+ /**
+ * Finds the proctime attribute if defined.
+ */
+ def deriveProctimeAttribute(properties: DescriptorProperties): Optional[String] = {
+ val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME)
+
+ for (i <- 0 until names.size) {
+ val isProctime = toScala(
+ properties.getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME"))
+ isProctime.foreach { isSet =>
+ if (isSet) {
+ return toJava(names.asScala.get(s"$SCHEMA.$i.$SCHEMA_NAME"))
+ }
+ }
+ }
+ toJava(None)
+ }
+
+ /**
+ * Finds the rowtime attributes if defined.
+ */
+ def deriveRowtimeAttributes(properties: DescriptorProperties)
+ : util.List[RowtimeAttributeDescriptor] = {
+
+ val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME)
+
+ var attributes = new mutable.ArrayBuffer[RowtimeAttributeDescriptor]()
+
+ // check for rowtime in every field
+ for (i <- 0 until names.size) {
+ RowtimeValidator
+ .getRowtimeComponents(properties, s"$SCHEMA.$i.")
+ .foreach { case (extractor, strategy) =>
+ // create descriptor
+ attributes += new RowtimeAttributeDescriptor(
+ properties.getString(s"$SCHEMA.$i.$SCHEMA_NAME"),
+ extractor,
+ strategy)
+ }
+ }
+
+ attributes.asJava
+ }
+
+ /**
+ * Finds a table source field mapping.
+ */
+ def deriveFieldMapping(
+ properties: DescriptorProperties,
+ sourceSchema: Optional[TableSchema])
+ : util.Map[String, String] = {
+
+ val mapping = mutable.Map[String, String]()
+
+ val schema = properties.getTableSchema(SCHEMA)
+
+ // add all schema fields first for implicit mappings
+ schema.getColumnNames.foreach { name =>
+ mapping.put(name, name)
+ }
+
+ val names = properties.getIndexedProperty(SCHEMA, SCHEMA_NAME)
+
+ for (i <- 0 until names.size) {
+ val name = properties.getString(s"$SCHEMA.$i.$SCHEMA_NAME")
+ toScala(properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM")) match {
- // per column properties
+ // add explicit mapping
+ case Some(source) =>
+ mapping.put(name, source)
- val NAME = "name"
- val TYPE = "type"
- val PROCTIME = "proctime"
- val PROCTIME_VALUE_TRUE = "true"
- val FROM = "from"
+ // implicit mapping or time
+ case None =>
+ val isProctime = properties
+ .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")
+ .orElse(false)
+ val isRowtime = properties
+ .containsKey(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE")
+ // remove proctime/rowtime from mapping
+ if (isProctime || isRowtime) {
+ mapping.remove(name)
+ }
+ // check for invalid fields
+ else if (toScala(sourceSchema).forall(s => !s.getColumnNames.contains(name))) {
+ throw new ValidationException(s"Could not map the schema field '$name' to a field " +
+ s"from source. Please specify the source field from which it can be derived.")
+ }
+ }
+ }
+ mapping.toMap.asJava
+ }
+
+ /**
+ * Finds the fields that can be used for a format schema (without time attributes).
+ */
+ def deriveFormatFields(properties: DescriptorProperties): TableSchema = {
+
+ val builder = TableSchema.builder()
+
+ val schema = properties.getTableSchema(SCHEMA)
+
+ schema.getColumnNames.zip(schema.getTypes).zipWithIndex.foreach { case ((n, t), i) =>
+ val isProctime = properties
+ .getOptionalBoolean(s"$SCHEMA.$i.$SCHEMA_PROCTIME")
+ .orElse(false)
+ val isRowtime = properties
+ .containsKey(s"$SCHEMA.$i.$ROWTIME_TIMESTAMPS_TYPE")
+ if (!isProctime && !isRowtime) {
+ // check for a aliasing
+ val fieldName = properties.getOptionalString(s"$SCHEMA.$i.$SCHEMA_FROM")
+ .orElse(n)
+ builder.field(fieldName, t)
+ }
+ }
+
+ builder.build()
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
index 3037286..f87a868 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/Statistics.scala
@@ -135,12 +135,12 @@ class Statistics extends Descriptor {
* Internal method for properties conversion.
*/
final override def addProperties(properties: DescriptorProperties): Unit = {
- properties.putInt(STATISTICS_VERSION, 1)
+ properties.putInt(STATISTICS_PROPERTY_VERSION, 1)
rowCount.foreach(rc => properties.putLong(STATISTICS_ROW_COUNT, rc))
val namedStats = columnStats.map { case (name, stats) =>
// name should not be part of the properties key
- (stats + (NAME -> name)).toMap
- }.toSeq
+ (stats + (NAME -> name)).toMap.asJava
+ }.toList.asJava
properties.putIndexedVariableProperties(STATISTICS_COLUMNS, namedStats)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
index a78e422..691cb21 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StatisticsValidator.scala
@@ -19,7 +19,8 @@
package org.apache.flink.table.descriptors
import org.apache.flink.table.api.ValidationException
-import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, STATISTICS_VERSION, validateColumnStats}
+import org.apache.flink.table.descriptors.DescriptorProperties.toScala
+import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_PROPERTY_VERSION, STATISTICS_ROW_COUNT, validateColumnStats}
import org.apache.flink.table.plan.stats.ColumnStats
import scala.collection.mutable
@@ -30,7 +31,7 @@ import scala.collection.mutable
class StatisticsValidator extends DescriptorValidator {
override def validate(properties: DescriptorProperties): Unit = {
- properties.validateInt(STATISTICS_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
+ properties.validateInt(STATISTICS_PROPERTY_VERSION, isOptional = true, 0, Integer.MAX_VALUE)
properties.validateLong(STATISTICS_ROW_COUNT, isOptional = true, min = 0)
validateColumnStats(properties, STATISTICS_COLUMNS)
}
@@ -38,7 +39,7 @@ class StatisticsValidator extends DescriptorValidator {
object StatisticsValidator {
- val STATISTICS_VERSION = "statistics.version"
+ val STATISTICS_PROPERTY_VERSION = "statistics.property-version"
val STATISTICS_ROW_COUNT = "statistics.row-count"
val STATISTICS_COLUMNS = "statistics.columns"
@@ -99,16 +100,16 @@ object StatisticsValidator {
val columnCount = properties.getIndexedProperty(key, NAME).size
val stats = for (i <- 0 until columnCount) yield {
- val name = properties.getString(s"$key.$i.$NAME").getOrElse(
+ val name = toScala(properties.getOptionalString(s"$key.$i.$NAME")).getOrElse(
throw new ValidationException(s"Could not find name of property '$key.$i.$NAME'."))
val stats = ColumnStats(
- properties.getLong(s"$key.$i.$DISTINCT_COUNT").map(v => Long.box(v)).orNull,
- properties.getLong(s"$key.$i.$NULL_COUNT").map(v => Long.box(v)).orNull,
- properties.getDouble(s"$key.$i.$AVG_LENGTH").map(v => Double.box(v)).orNull,
- properties.getInt(s"$key.$i.$MAX_LENGTH").map(v => Int.box(v)).orNull,
- properties.getDouble(s"$key.$i.$MAX_VALUE").map(v => Double.box(v)).orNull,
- properties.getDouble(s"$key.$i.$MIN_VALUE").map(v => Double.box(v)).orNull
+ properties.getOptionalLong(s"$key.$i.$DISTINCT_COUNT").orElse(null),
+ properties.getOptionalLong(s"$key.$i.$NULL_COUNT").orElse(null),
+ properties.getOptionalDouble(s"$key.$i.$AVG_LENGTH").orElse(null),
+ properties.getOptionalInt(s"$key.$i.$MAX_LENGTH").orElse(null),
+ properties.getOptionalDouble(s"$key.$i.$MAX_VALUE").orElse(null),
+ properties.getOptionalDouble(s"$key.$i.$MIN_VALUE").orElse(null)
)
name -> stats
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala
index 5e0b42a..8f2e473 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/StreamTableSourceDescriptor.scala
@@ -46,7 +46,7 @@ class StreamTableSourceDescriptor(tableEnv: StreamTableEnvironment, connector: C
* Searches for the specified table source, configures it accordingly, and returns it.
*/
def toTableSource: TableSource[_] = {
- val source = TableSourceFactoryService.findTableSourceFactory(this)
+ val source = TableSourceFactoryService.findAndCreateTableSource(this)
source match {
case _: StreamTableSource[_] => source
case _ => throw new TableException(
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala
index a49a41b..5118489 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/descriptors/TableSourceDescriptor.scala
@@ -18,6 +18,7 @@
package org.apache.flink.table.descriptors
+import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.StatisticsValidator.{STATISTICS_COLUMNS, STATISTICS_ROW_COUNT, readColumnStats}
import org.apache.flink.table.plan.stats.TableStats
@@ -50,7 +51,7 @@ abstract class TableSourceDescriptor extends Descriptor {
protected def getTableStats: Option[TableStats] = {
val normalizedProps = new DescriptorProperties()
addProperties(normalizedProps)
- val rowCount = normalizedProps.getLong(STATISTICS_ROW_COUNT).map(v => Long.box(v))
+ val rowCount = toScala(normalizedProps.getOptionalLong(STATISTICS_ROW_COUNT))
rowCount match {
case Some(cnt) =>
val columnStats = readColumnStats(normalizedProps, STATISTICS_COLUMNS)
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala
index bec4565..06d6bfb 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/CsvTableSourceFactory.scala
@@ -21,11 +21,12 @@ package org.apache.flink.table.sources
import java.util
import org.apache.flink.table.api.TableException
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_TYPE, CONNECTOR_VERSION}
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.{CONNECTOR_PROPERTY_VERSION, CONNECTOR_TYPE}
import org.apache.flink.table.descriptors.CsvValidator._
+import org.apache.flink.table.descriptors.DescriptorProperties.toScala
import org.apache.flink.table.descriptors.FileSystemValidator.{CONNECTOR_PATH, CONNECTOR_TYPE_VALUE}
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_TYPE, FORMAT_VERSION}
-import org.apache.flink.table.descriptors.SchemaValidator.{SCHEMA, SCHEMA_VERSION}
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.{FORMAT_PROPERTY_VERSION, FORMAT_TYPE}
+import org.apache.flink.table.descriptors.SchemaValidator.SCHEMA
import org.apache.flink.table.descriptors._
import org.apache.flink.types.Row
@@ -38,9 +39,8 @@ class CsvTableSourceFactory extends TableSourceFactory[Row] {
val context = new util.HashMap[String, String]()
context.put(CONNECTOR_TYPE, CONNECTOR_TYPE_VALUE)
context.put(FORMAT_TYPE, FORMAT_TYPE_VALUE)
- context.put(CONNECTOR_VERSION, "1")
- context.put(FORMAT_VERSION, "1")
- context.put(SCHEMA_VERSION, "1")
+ context.put(CONNECTOR_PROPERTY_VERSION, "1")
+ context.put(FORMAT_PROPERTY_VERSION, "1")
context
}
@@ -76,33 +76,36 @@ class CsvTableSourceFactory extends TableSourceFactory[Row] {
// build
val csvTableSourceBuilder = new CsvTableSource.Builder
- val tableSchema = params.getTableSchema(SCHEMA).get
- val encodingSchema = params.getTableSchema(FORMAT_FIELDS)
+ val formatSchema = params.getTableSchema(FORMAT_FIELDS)
+ val tableSchema = params.getTableSchema(SCHEMA)
// the CsvTableSource needs some rework first
// for now the schema must be equal to the encoding
- if (!encodingSchema.contains(tableSchema)) {
+ if (!formatSchema.equals(tableSchema)) {
throw new TableException(
"Encodings that differ from the schema are not supported yet for CsvTableSources.")
}
- params.getString(CONNECTOR_PATH).foreach(csvTableSourceBuilder.path)
- params.getString(FORMAT_FIELD_DELIMITER).foreach(csvTableSourceBuilder.fieldDelimiter)
- params.getString(FORMAT_LINE_DELIMITER).foreach(csvTableSourceBuilder.lineDelimiter)
+ toScala(params.getOptionalString(CONNECTOR_PATH))
+ .foreach(csvTableSourceBuilder.path)
+ toScala(params.getOptionalString(FORMAT_FIELD_DELIMITER))
+ .foreach(csvTableSourceBuilder.fieldDelimiter)
+ toScala(params.getOptionalString(FORMAT_LINE_DELIMITER))
+ .foreach(csvTableSourceBuilder.lineDelimiter)
- encodingSchema.foreach { schema =>
- schema.getColumnNames.zip(schema.getTypes).foreach { case (name, tpe) =>
- csvTableSourceBuilder.field(name, tpe)
- }
+ formatSchema.getColumnNames.zip(formatSchema.getTypes).foreach { case (name, tpe) =>
+ csvTableSourceBuilder.field(name, tpe)
}
- params.getCharacter(FORMAT_QUOTE_CHARACTER).foreach(csvTableSourceBuilder.quoteCharacter)
- params.getString(FORMAT_COMMENT_PREFIX).foreach(csvTableSourceBuilder.commentPrefix)
- params.getBoolean(FORMAT_IGNORE_FIRST_LINE).foreach { flag =>
+ toScala(params.getOptionalCharacter(FORMAT_QUOTE_CHARACTER))
+ .foreach(csvTableSourceBuilder.quoteCharacter)
+ toScala(params.getOptionalString(FORMAT_COMMENT_PREFIX))
+ .foreach(csvTableSourceBuilder.commentPrefix)
+ toScala(params.getOptionalBoolean(FORMAT_IGNORE_FIRST_LINE)).foreach { flag =>
if (flag) {
csvTableSourceBuilder.ignoreFirstLine()
}
}
- params.getBoolean(FORMAT_IGNORE_PARSE_ERRORS).foreach { flag =>
+ toScala(params.getOptionalBoolean(FORMAT_IGNORE_PARSE_ERRORS)).foreach { flag =>
if (flag) {
csvTableSourceBuilder.ignoreParseErrors()
}
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala
index f42d765..e5f6965 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactory.scala
@@ -41,10 +41,10 @@ trait TableSourceFactory[T] {
* - connector.type
* - format.type
*
- * Specified versions allow the framework to provide backwards compatible properties in case of
- * string format changes:
- * - connector.version
- * - format.version
+ * Specified property versions allow the framework to provide backwards compatible properties
+ * in case of string format changes:
+ * - connector.property-version
+ * - format.property-version
*
* An empty context means that the factory matches for all requests.
*/
@@ -61,7 +61,8 @@ trait TableSourceFactory[T] {
* - format.fields.#.type
* - format.fields.#.name
*
- * Note: Use "#" to denote an array of values where "#" represents one or more digits.
+ * Note: Use "#" to denote an array of values where "#" represents one or more digits. Property
+ * versions like "format.property-version" must not be part of the supported properties.
*/
def supportedProperties(): util.List[String]
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala
index 1e8e836..877cb7b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/TableSourceFactoryService.scala
@@ -21,12 +21,10 @@ package org.apache.flink.table.sources
import java.util.{ServiceConfigurationError, ServiceLoader}
import org.apache.flink.table.api.{AmbiguousTableSourceException, NoMatchingTableSourceException, TableException, ValidationException}
-import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION
-import org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_VERSION
-import org.apache.flink.table.descriptors.MetadataValidator.METADATA_VERSION
-import org.apache.flink.table.descriptors.RowtimeValidator.ROWTIME_VERSION
-import org.apache.flink.table.descriptors.SchemaValidator.SCHEMA_VERSION
-import org.apache.flink.table.descriptors.StatisticsValidator.STATISTICS_VERSION
+import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_PROPERTY_VERSION
+import org.apache.flink.table.descriptors.FormatDescriptorValidator.FORMAT_PROPERTY_VERSION
+import org.apache.flink.table.descriptors.MetadataValidator.METADATA_PROPERTY_VERSION
+import org.apache.flink.table.descriptors.StatisticsValidator.STATISTICS_PROPERTY_VERSION
import org.apache.flink.table.descriptors._
import org.apache.flink.table.util.Logging
@@ -40,13 +38,13 @@ object TableSourceFactoryService extends Logging {
private lazy val loader = ServiceLoader.load(classOf[TableSourceFactory[_]])
- def findTableSourceFactory(descriptor: TableSourceDescriptor): TableSource[_] = {
+ def findAndCreateTableSource(descriptor: TableSourceDescriptor): TableSource[_] = {
val properties = new DescriptorProperties()
descriptor.addProperties(properties)
- findTableSourceFactory(properties.asMap)
+ findAndCreateTableSource(properties.asMap.asScala.toMap)
}
- def findTableSourceFactory(properties: Map[String, String]): TableSource[_] = {
+ def findAndCreateTableSource(properties: Map[String, String]): TableSource[_] = {
var matchingFactory: Option[(TableSourceFactory[_], Seq[String])] = None
try {
val iter = loader.iterator()
@@ -73,12 +71,10 @@ object TableSourceFactoryService extends Logging {
plainContext ++= requiredContext
// we remove the versions for now until we have the first backwards compatibility case
// with the version we can provide mappings in case the format changes
- plainContext.remove(CONNECTOR_VERSION)
- plainContext.remove(FORMAT_VERSION)
- plainContext.remove(SCHEMA_VERSION)
- plainContext.remove(ROWTIME_VERSION)
- plainContext.remove(METADATA_VERSION)
- plainContext.remove(STATISTICS_VERSION)
+ plainContext.remove(CONNECTOR_PROPERTY_VERSION)
+ plainContext.remove(FORMAT_PROPERTY_VERSION)
+ plainContext.remove(METADATA_PROPERTY_VERSION)
+ plainContext.remove(STATISTICS_PROPERTY_VERSION)
// check if required context is met
if (plainContext.forall(e => properties.contains(e._1) && properties(e._1) == e._2)) {
http://git-wip-us.apache.org/repos/asf/flink/blob/db2c510f/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala
index 329f790..fcbd63f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/sources/tsextractors/StreamRecordTimestamp.scala
@@ -27,7 +27,7 @@ import org.apache.flink.table.expressions.{Expression, ResolvedFieldReference}
*
* Note: This extractor only works for StreamTableSources.
*/
-class StreamRecordTimestamp extends TimestampExtractor {
+final class StreamRecordTimestamp extends TimestampExtractor {
/** No argument fields required. */
override def getArgumentFields: Array[String] = Array()
@@ -42,5 +42,8 @@ class StreamRecordTimestamp extends TimestampExtractor {
override def getExpression(fieldAccesses: Array[ResolvedFieldReference]): Expression = {
org.apache.flink.table.expressions.StreamRecordTimestamp()
}
+}
+object StreamRecordTimestamp {
+ val INSTANCE: StreamRecordTimestamp = new StreamRecordTimestamp
}