You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ti...@apache.org on 2023/08/31 02:50:23 UTC
[flink-connector-pulsar] branch main updated: [FLINK-26203] Basic table factory for Pulsar connector (#56)
This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-pulsar.git
The following commit(s) were added to refs/heads/main by this push:
new 75c79ad [FLINK-26203] Basic table factory for Pulsar connector (#56)
75c79ad is described below
commit 75c79ad09ea43e44e223248174f6d80c09832b7c
Author: tison <wa...@gmail.com>
AuthorDate: Thu Aug 31 10:50:18 2023 +0800
[FLINK-26203] Basic table factory for Pulsar connector (#56)
Signed-off-by: tison <wa...@gmail.com>
Co-author-by: Christophe Bornet <cb...@hotmail.com>
Co-author-by: Yufei Zhang <af...@gmail.com>
Co-author-by: Yufan Sheng <yu...@streamnative.io>
---
flink-connector-pulsar/pom.xml | 49 ++
.../connector/pulsar/sink/writer/PulsarWriter.java | 9 +
.../pulsar/sink/writer/message/PulsarMessage.java | 7 +
.../sink/writer/message/PulsarMessageBuilder.java | 14 +
.../cursor/start/MessageIdStartCursor.java | 19 +
.../cursor/start/TimestampStartCursor.java | 19 +
.../cursor/stop/EventTimestampStopCursor.java | 19 +
.../cursor/stop/LatestMessageStopCursor.java | 19 +
.../cursor/stop/MessageIdStopCursor.java | 19 +
.../enumerator/cursor/stop/NeverStopCursor.java | 16 +-
.../cursor/stop/PublishTimestampStopCursor.java | 19 +
.../connector/pulsar/table/PulsarTableFactory.java | 301 +++++++++
.../pulsar/table/PulsarTableOptionUtils.java | 324 ++++++++++
.../connector/pulsar/table/PulsarTableOptions.java | 285 +++++++++
.../pulsar/table/PulsarTableValidationUtils.java | 196 ++++++
.../pulsar/table/UpsertPulsarTableFactory.java | 314 ++++++++++
.../table/sink/PulsarTableSerializationSchema.java | 132 ++++
.../PulsarTableSerializationSchemaFactory.java | 160 +++++
.../pulsar/table/sink/PulsarTableSink.java | 174 ++++++
.../pulsar/table/sink/PulsarWritableMetadata.java | 140 +++++
.../table/source/PulsarReadableMetadata.java | 145 +++++
.../table/source/PulsarRowDataConverter.java | 134 ++++
.../source/PulsarTableDeserializationSchema.java | 116 ++++
.../PulsarTableDeserializationSchemaFactory.java | 215 +++++++
.../pulsar/table/source/PulsarTableSource.java | 226 +++++++
.../org.apache.flink.table.factories.Factory | 17 +
.../pulsar/table/PulsarChangelogTableITCase.java | 458 ++++++++++++++
.../pulsar/table/PulsarTableFactoryTest.java | 440 +++++++++++++
.../connector/pulsar/table/PulsarTableITCase.java | 678 +++++++++++++++++++++
.../pulsar/table/PulsarTableOptionUtilsTest.java | 416 +++++++++++++
.../pulsar/table/PulsarTableOptionsTest.java | 496 +++++++++++++++
.../pulsar/table/PulsarTableTestBase.java | 76 +++
.../table/PulsarTableValidationUtilsTest.java | 183 ++++++
.../pulsar/table/UpsertPulsarTableITCase.java | 573 +++++++++++++++++
.../table/testutils/MockPulsarAuthentication.java | 84 +++
.../pulsar/table/testutils/MockTopicRouter.java} | 27 +-
.../table/testutils/PulsarTableTestUtils.java | 114 ++++
.../pulsar/table/testutils/TestingUser.java | 61 ++
.../testutils/runtime/PulsarRuntimeOperator.java | 4 +-
.../src/test/resources/canal-data.txt | 11 +
.../resources/debezium-data-schema-exclude.txt | 16 +
.../src/test/resources/log4j2-test.properties | 6 +-
.../src/test/resources/maxwell-data.txt | 20 +
pom.xml | 17 +
44 files changed, 6750 insertions(+), 18 deletions(-)
diff --git a/flink-connector-pulsar/pom.xml b/flink-connector-pulsar/pom.xml
index ff36209..96d96a9 100644
--- a/flink-connector-pulsar/pom.xml
+++ b/flink-connector-pulsar/pom.xml
@@ -51,6 +51,17 @@ under the License.
<scope>provided</scope>
</dependency>
+ <!-- Table ecosystem -->
+
+ <!-- Projects depending on this project won't depend on flink-table-*. -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java-bridge</artifactId>
+ <version>${flink.version}</version>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
<!-- Formats -->
<dependency>
@@ -107,6 +118,36 @@ under the License.
<scope>test</scope>
</dependency>
+ <!-- Table factory testing -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${flink.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-test-utils</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-scala-bridge_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
<!-- Pulsar testing environment -->
<dependency>
@@ -114,6 +155,14 @@ under the License.
<artifactId>pulsar</artifactId>
<scope>test</scope>
</dependency>
+
+ <!-- Pulsar SQL IT test with formats -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-csv</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
index a1de8a2..5c93ef3 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/PulsarWriter.java
@@ -44,6 +44,7 @@ import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.TypedMessageBuilder;
+import org.apache.pulsar.client.impl.TypedMessageBuilderImpl;
import org.apache.pulsar.shade.com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -199,6 +200,14 @@ public class PulsarWriter<IN> implements PrecommittingSinkWriter<IN, PulsarCommi
builder.key(key);
}
+ if (message.isBase64EncodedKey()) {
+ // HACK - otherwise we should hold both keys and keyBytes fields which
+ // is more confusing.
+ ((TypedMessageBuilderImpl<?>) builder)
+ .getMetadataBuilder()
+ .setPartitionKeyB64Encoded(true);
+ }
+
long eventTime = message.getEventTime();
if (eventTime > 0) {
builder.eventTime(eventTime);
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
index f33766e..64a4484 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessage.java
@@ -39,6 +39,7 @@ public class PulsarMessage<T> {
@Nullable private final byte[] orderingKey;
@Nullable private final String key;
+ private final boolean isBase64EncodedKey;
private final long eventTime;
@Nullable private final Schema<T> schema;
@Nullable private final T value;
@@ -57,6 +58,7 @@ public class PulsarMessage<T> {
PulsarMessage(
@Nullable byte[] orderingKey,
@Nullable String key,
+ boolean isBase64EncodedKey,
long eventTime,
@Nullable Schema<T> schema,
@Nullable T value,
@@ -66,6 +68,7 @@ public class PulsarMessage<T> {
boolean disableReplication) {
this.orderingKey = orderingKey;
this.key = key;
+ this.isBase64EncodedKey = isBase64EncodedKey;
this.eventTime = eventTime;
this.schema = schema;
this.value = value;
@@ -117,6 +120,10 @@ public class PulsarMessage<T> {
return key;
}
+ public boolean isBase64EncodedKey() {
+ return isBase64EncodedKey;
+ }
+
public long getEventTime() {
return eventTime;
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
index ac48918..585f885 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/sink/writer/message/PulsarMessageBuilder.java
@@ -26,6 +26,7 @@ import org.apache.pulsar.client.api.TypedMessageBuilder;
import javax.annotation.Nullable;
+import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -39,6 +40,7 @@ public class PulsarMessageBuilder<T> {
private byte[] orderingKey;
private String key;
+ private boolean isBase64EncodedKey;
private long eventTime;
@Nullable private final Schema<T> schema;
@Nullable private final T value;
@@ -68,6 +70,17 @@ public class PulsarMessageBuilder<T> {
*/
public PulsarMessageBuilder<T> key(String key) {
this.key = checkNotNull(key);
+ this.isBase64EncodedKey = false;
+ return this;
+ }
+
+ /**
+ * Method wrapper of {@link TypedMessageBuilder#keyBytes(byte[])}. This key would also be used
+ * in {@link KeyHashTopicRouter}.
+ */
+ public PulsarMessageBuilder<T> keyBytes(byte[] keyBytes) {
+ this.key = Base64.getEncoder().encodeToString(keyBytes);
+ this.isBase64EncodedKey = true;
return this;
}
@@ -115,6 +128,7 @@ public class PulsarMessageBuilder<T> {
return new PulsarMessage<>(
orderingKey,
key,
+ isBase64EncodedKey,
eventTime,
schema,
value,
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/MessageIdStartCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/MessageIdStartCursor.java
index 23dd98e..b180262 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/MessageIdStartCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/MessageIdStartCursor.java
@@ -24,6 +24,8 @@ import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.MessageId;
+import java.util.Objects;
+
/** This cursor would leave pulsar start consuming from a specific message id. */
public class MessageIdStartCursor implements StartCursor {
private static final long serialVersionUID = -8057345435887170111L;
@@ -50,4 +52,21 @@ public class MessageIdStartCursor implements StartCursor {
public CursorPosition position(String topic, int partitionId) {
return new CursorPosition(messageId, inclusive);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MessageIdStartCursor that = (MessageIdStartCursor) o;
+ return Objects.equals(messageId, that.messageId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(messageId);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java
index da51a58..065de66 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java
@@ -21,6 +21,8 @@ package org.apache.flink.connector.pulsar.source.enumerator.cursor.start;
import org.apache.flink.connector.pulsar.source.enumerator.cursor.CursorPosition;
import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import java.util.Objects;
+
/** This cursor would left pulsar start consuming from a specific publish timestamp. */
public class TimestampStartCursor implements StartCursor {
private static final long serialVersionUID = 5170578885838095320L;
@@ -35,4 +37,21 @@ public class TimestampStartCursor implements StartCursor {
public CursorPosition position(String topic, int partitionId) {
return new CursorPosition(timestamp);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TimestampStartCursor that = (TimestampStartCursor) o;
+ return timestamp == that.timestamp;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/EventTimestampStopCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/EventTimestampStopCursor.java
index d2a44ea..dfaadbe 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/EventTimestampStopCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/EventTimestampStopCursor.java
@@ -22,6 +22,8 @@ import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
import org.apache.pulsar.client.api.Message;
+import java.util.Objects;
+
/** Stop consuming message at the given event time. */
public class EventTimestampStopCursor implements StopCursor {
private static final long serialVersionUID = 2391576769339369027L;
@@ -39,4 +41,21 @@ public class EventTimestampStopCursor implements StopCursor {
long eventTime = message.getEventTime();
return StopCondition.compare(timestamp, eventTime, inclusive);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ EventTimestampStopCursor that = (EventTimestampStopCursor) o;
+ return timestamp == that.timestamp && inclusive == that.inclusive;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp, inclusive);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/LatestMessageStopCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/LatestMessageStopCursor.java
index 5af8be8..def12b1 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/LatestMessageStopCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/LatestMessageStopCursor.java
@@ -26,6 +26,8 @@ import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
+import java.util.Objects;
+
/**
* A stop cursor that initialize the position to the latest message id. The offsets initialization
* are taken care of by the {@code PulsarPartitionSplitReaderBase} instead of by the {@code
@@ -54,4 +56,21 @@ public class LatestMessageStopCursor implements StopCursor {
this.messageId = admin.topics().getLastMessageId(topic);
}
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ LatestMessageStopCursor that = (LatestMessageStopCursor) o;
+ return inclusive == that.inclusive && Objects.equals(messageId, that.messageId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(messageId, inclusive);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/MessageIdStopCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/MessageIdStopCursor.java
index 668b8ef..43a7716 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/MessageIdStopCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/MessageIdStopCursor.java
@@ -23,6 +23,8 @@ import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
+import java.util.Objects;
+
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.pulsar.client.api.MessageId.earliest;
import static org.apache.pulsar.client.api.MessageId.latest;
@@ -51,4 +53,21 @@ public class MessageIdStopCursor implements StopCursor {
MessageId current = message.getMessageId();
return StopCondition.compare(messageId, current, inclusive);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MessageIdStopCursor that = (MessageIdStopCursor) o;
+ return inclusive == that.inclusive && Objects.equals(messageId, that.messageId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(messageId, inclusive);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/NeverStopCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/NeverStopCursor.java
index 3eb0356..94e5b31 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/NeverStopCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/NeverStopCursor.java
@@ -22,7 +22,7 @@ import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
import org.apache.pulsar.client.api.Message;
-/** A implementation which wouldn't stop forever. */
+/** An implementation which wouldn't stop forever. */
public class NeverStopCursor implements StopCursor {
private static final long serialVersionUID = -3113601090292771786L;
@@ -30,4 +30,18 @@ public class NeverStopCursor implements StopCursor {
public StopCondition shouldStop(Message<?> message) {
return StopCondition.CONTINUE;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ return o != null && getClass() == o.getClass();
+ }
+
+ @Override
+ public int hashCode() {
+ return 31;
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/PublishTimestampStopCursor.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/PublishTimestampStopCursor.java
index 2dfdd76..c374f90 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/PublishTimestampStopCursor.java
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/stop/PublishTimestampStopCursor.java
@@ -22,6 +22,8 @@ import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
import org.apache.pulsar.client.api.Message;
+import java.util.Objects;
+
/** Stop consuming message at the given publish time. */
public class PublishTimestampStopCursor implements StopCursor {
private static final long serialVersionUID = 4386276745339324527L;
@@ -39,4 +41,21 @@ public class PublishTimestampStopCursor implements StopCursor {
long publishTime = message.getPublishTime();
return StopCondition.compare(timestamp, publishTime, inclusive);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PublishTimestampStopCursor that = (PublishTimestampStopCursor) o;
+ return timestamp == that.timestamp && inclusive == that.inclusive;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timestamp, inclusive);
+ }
}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableFactory.java
new file mode 100644
index 0000000..b867e18
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableFactory.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.pulsar.common.config.PulsarOptions;
+import org.apache.flink.connector.pulsar.sink.PulsarSinkOptions;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.source.PulsarSourceOptions;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSerializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSink;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableDeserializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableSource;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createKeyFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createValueFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getKeyDecodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getKeyEncodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getMessageDelayMillis;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getPulsarProperties;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStartCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStopCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getSubscriptionType;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicListFromOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicRouter;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicRoutingMode;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getValueDecodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getValueEncodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.ADMIN_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.EXPLICIT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SERVICE_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_MESSAGE_DELAY_INTERVAL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AFTER_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.VALUE_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validatePrimaryKeyConstraints;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateTableSinkOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateTableSourceOptions;
+import static org.apache.flink.table.factories.FactoryUtil.SINK_PARALLELISM;
+import static org.apache.pulsar.shade.org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+
+/**
+ * Factory for creating {@link DynamicTableSource} and {@link DynamicTableSink}.
+ *
+ * <p>The main role of this class is to retrieve config options and validate options from config and
+ * the table schema. It also sets default values if a config option is not present.
+ */
+public class PulsarTableFactory implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+
+ public static final String IDENTIFIER = "pulsar";
+
+ public static final String DEFAULT_SUBSCRIPTION_NAME_PREFIX = "flink-sql-connector-pulsar-";
+
+ public static final boolean UPSERT_DISABLED = false;
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+ // Format options should be retrieved before validation.
+ final DecodingFormat<DeserializationSchema<RowData>> keyDecodingFormat =
+ getKeyDecodingFormat(helper);
+ final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat =
+ getValueDecodingFormat(helper);
+ ReadableConfig tableOptions = helper.getOptions();
+
+ // Validate configs are not conflict; each options is consumed; no unwanted configs
+ // PulsarOptions, PulsarSourceOptions and PulsarSinkOptions is not part of the validation.
+ helper.validateExcept(
+ PulsarOptions.CLIENT_CONFIG_PREFIX,
+ PulsarOptions.ADMIN_CONFIG_PREFIX,
+ PulsarSourceOptions.SOURCE_CONFIG_PREFIX,
+ PulsarSourceOptions.CONSUMER_CONFIG_PREFIX,
+ PulsarSinkOptions.PRODUCER_CONFIG_PREFIX,
+ PulsarSinkOptions.SINK_CONFIG_PREFIX);
+
+ validatePrimaryKeyConstraints(
+ context.getObjectIdentifier(), context.getPrimaryKeyIndexes(), helper);
+
+ validateTableSourceOptions(tableOptions);
+
+ // Retrieve configs
+ final List<String> topics = getTopicListFromOptions(tableOptions);
+ final StartCursor startCursor = getStartCursor(tableOptions);
+ final StopCursor stopCursor = getStopCursor(tableOptions);
+ final SubscriptionType subscriptionType = getSubscriptionType(tableOptions);
+
+ // Forward source configs
+ final Properties properties = getPulsarProperties(tableOptions);
+ properties.setProperty(PULSAR_ADMIN_URL.key(), tableOptions.get(ADMIN_URL));
+ properties.setProperty(PULSAR_SERVICE_URL.key(), tableOptions.get(SERVICE_URL));
+ // Set random subscriptionName if not provided
+ properties.setProperty(
+ PULSAR_SUBSCRIPTION_NAME.key(),
+ tableOptions
+ .getOptional(SOURCE_SUBSCRIPTION_NAME)
+ .orElse(DEFAULT_SUBSCRIPTION_NAME_PREFIX + randomAlphabetic(5)));
+ // Retrieve physical fields (not including computed or metadata fields),
+ // and projections and create a schema factory based on such information.
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+
+ final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
+ final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
+
+ final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory =
+ new PulsarTableDeserializationSchemaFactory(
+ physicalDataType,
+ keyDecodingFormat,
+ keyProjection,
+ valueDecodingFormat,
+ valueProjection,
+ UPSERT_DISABLED);
+
+ // Set default values for configuration not exposed to user.
+ final DecodingFormat<DeserializationSchema<RowData>> decodingFormatForMetadataPushdown =
+ valueDecodingFormat;
+ final ChangelogMode changelogMode = decodingFormatForMetadataPushdown.getChangelogMode();
+
+ return new PulsarTableSource(
+ deserializationSchemaFactory,
+ decodingFormatForMetadataPushdown,
+ changelogMode,
+ topics,
+ properties,
+ startCursor,
+ stopCursor,
+ subscriptionType);
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+ // Format options should be retrieved before validation.
+ final EncodingFormat<SerializationSchema<RowData>> keyEncodingFormat =
+ getKeyEncodingFormat(helper);
+ final EncodingFormat<SerializationSchema<RowData>> valueEncodingFormat =
+ getValueEncodingFormat(helper);
+ ReadableConfig tableOptions = helper.getOptions();
+
+ // Validate configs are not conflict; each options is consumed; no unwanted configs
+ // PulsarOptions, PulsarSourceOptions and PulsarSinkOptions is not part of the validation.
+ helper.validateExcept(
+ PulsarOptions.CLIENT_CONFIG_PREFIX,
+ PulsarOptions.ADMIN_CONFIG_PREFIX,
+ PulsarSourceOptions.SOURCE_CONFIG_PREFIX,
+ PulsarSourceOptions.CONSUMER_CONFIG_PREFIX,
+ PulsarSinkOptions.PRODUCER_CONFIG_PREFIX,
+ PulsarSinkOptions.SINK_CONFIG_PREFIX);
+
+ validatePrimaryKeyConstraints(
+ context.getObjectIdentifier(), context.getPrimaryKeyIndexes(), helper);
+
+ validateTableSinkOptions(tableOptions);
+
+ // Retrieve configs
+ final TopicRouter<RowData> topicRouter =
+ getTopicRouter(tableOptions, context.getClassLoader());
+ final TopicRoutingMode topicRoutingMode = getTopicRoutingMode(tableOptions);
+ final long messageDelayMillis = getMessageDelayMillis(tableOptions);
+
+ final List<String> topics = getTopicListFromOptions(tableOptions);
+
+ // Forward sink configs
+ final Properties properties = getPulsarProperties(tableOptions);
+ properties.setProperty(PULSAR_ADMIN_URL.key(), tableOptions.get(ADMIN_URL));
+ properties.setProperty(PULSAR_SERVICE_URL.key(), tableOptions.get(SERVICE_URL));
+
+ // Retrieve physical DataType (not including computed or metadata fields)
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+ final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
+ final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
+
+ final PulsarTableSerializationSchemaFactory serializationSchemaFactory =
+ new PulsarTableSerializationSchemaFactory(
+ physicalDataType,
+ keyEncodingFormat,
+ keyProjection,
+ valueEncodingFormat,
+ valueProjection,
+ UPSERT_DISABLED);
+
+ // Set default values for configuration not exposed to user.
+ final DeliveryGuarantee deliveryGuarantee = DeliveryGuarantee.AT_LEAST_ONCE;
+ final ChangelogMode changelogMode = valueEncodingFormat.getChangelogMode();
+
+ return new PulsarTableSink(
+ serializationSchemaFactory,
+ changelogMode,
+ topics,
+ properties,
+ deliveryGuarantee,
+ topicRouter,
+ topicRoutingMode,
+ messageDelayMillis);
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Stream.of(TOPICS, ADMIN_URL, SERVICE_URL).collect(Collectors.toSet());
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return Stream.of(
+ FactoryUtil.FORMAT,
+ VALUE_FORMAT,
+ SOURCE_SUBSCRIPTION_NAME,
+ SOURCE_SUBSCRIPTION_TYPE,
+ SOURCE_START_FROM_MESSAGE_ID,
+ SOURCE_START_FROM_PUBLISH_TIME,
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME,
+ SINK_CUSTOM_TOPIC_ROUTER,
+ SINK_TOPIC_ROUTING_MODE,
+ SINK_MESSAGE_DELAY_INTERVAL,
+ SINK_PARALLELISM,
+ KEY_FORMAT,
+ KEY_FIELDS,
+ EXPLICIT)
+ .collect(Collectors.toSet());
+ }
+
+ /** Format and Delivery guarantee related options are not forward options. */
+ @Override
+ public Set<ConfigOption<?>> forwardOptions() {
+ return Stream.of(
+ TOPICS,
+ ADMIN_URL,
+ SERVICE_URL,
+ SOURCE_SUBSCRIPTION_TYPE,
+ SOURCE_SUBSCRIPTION_NAME,
+ SOURCE_START_FROM_MESSAGE_ID,
+ SOURCE_START_FROM_PUBLISH_TIME,
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME,
+ SINK_CUSTOM_TOPIC_ROUTER,
+ SINK_TOPIC_ROUTING_MODE,
+ SINK_MESSAGE_DELAY_INTERVAL)
+ .collect(Collectors.toSet());
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtils.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtils.java
new file mode 100644
index 0000000..8227bf2
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtils.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DeserializationFormatFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.SerializationFormatFactory;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.InstantiationUtil;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_MESSAGE_DELAY_INTERVAL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AFTER_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.VALUE_FORMAT;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * A util class for getting fields from config options, getting formats and other useful
+ * information.
+ *
+ * <p>It contains the following functionalities.
+ *
+ * <ul>
+ * <li>Get Topics from configurations.
+ * <li>Get source StartCursor from configurations.
+ * <li>Get source SubscriptionType from configurations.
+ * <li>Get sink messageDelayMillis from configurations.
+ * <li>Get sink TopicRouter/TopicRoutingMode from configurations.
+ * <li>Create key and value encoding/decoding format.
+ * <li>Create key and value projection.
+ * </ul>
+ */
+public class PulsarTableOptionUtils {
+
+ private PulsarTableOptionUtils() {}
+
+ public static final String TOPIC_LIST_DELIMITER = ";";
+
+ // --------------------------------------------------------------------------------------------
+ // Decoding / Encoding and Projection
+ // --------------------------------------------------------------------------------------------
+
+ @Nullable
+ public static DecodingFormat<DeserializationSchema<RowData>> getKeyDecodingFormat(
+ FactoryUtil.TableFactoryHelper helper) {
+ return helper.discoverOptionalDecodingFormat(DeserializationFormatFactory.class, KEY_FORMAT)
+ .orElse(null);
+ }
+
+ @Nullable
+ public static EncodingFormat<SerializationSchema<RowData>> getKeyEncodingFormat(
+ FactoryUtil.TableFactoryHelper helper) {
+ return helper.discoverOptionalEncodingFormat(SerializationFormatFactory.class, KEY_FORMAT)
+ .orElse(null);
+ }
+
+ public static DecodingFormat<DeserializationSchema<RowData>> getValueDecodingFormat(
+ FactoryUtil.TableFactoryHelper helper) {
+ return helper.discoverOptionalDecodingFormat(
+ DeserializationFormatFactory.class, FactoryUtil.FORMAT)
+ .orElseGet(
+ () ->
+ helper.discoverDecodingFormat(
+ DeserializationFormatFactory.class, VALUE_FORMAT));
+ }
+
+ public static EncodingFormat<SerializationSchema<RowData>> getValueEncodingFormat(
+ FactoryUtil.TableFactoryHelper helper) {
+ return helper.discoverOptionalEncodingFormat(
+ SerializationFormatFactory.class, FactoryUtil.FORMAT)
+ .orElseGet(
+ () ->
+ helper.discoverEncodingFormat(
+ SerializationFormatFactory.class, VALUE_FORMAT));
+ }
+
+ /**
+ * Creates an array of indices that determine which physical fields of the table schema to
+ * include in the key format and the order that those fields have in the key format.
+ */
+ public static int[] createKeyFormatProjection(
+ ReadableConfig options, DataType physicalDataType) {
+ final LogicalType physicalType = physicalDataType.getLogicalType();
+ checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
+ final Optional<String> optionalKeyFormat = options.getOptional(KEY_FORMAT);
+ final Optional<List<String>> optionalKeyFields = options.getOptional(KEY_FIELDS);
+
+ if (!optionalKeyFormat.isPresent()) {
+ return new int[0];
+ }
+
+ final List<String> keyFields = optionalKeyFields.get();
+ final List<String> physicalFields = LogicalTypeChecks.getFieldNames(physicalType);
+ return keyFields.stream()
+ .mapToInt(
+ keyField -> {
+ final int pos = physicalFields.indexOf(keyField);
+ // check that field name exists
+ if (pos < 0) {
+ throw new ValidationException(
+ String.format(
+ "Could not find the field '%s' in the table schema for usage in the key format. "
+ + "A key field must be a regular, physical column. "
+ + "The following columns can be selected in the '%s' option: %s",
+ keyField, KEY_FIELDS.key(), physicalFields));
+ }
+ // check that field name is prefixed correctly
+ return pos;
+ })
+ .toArray();
+ }
+
+ public static int[] createValueFormatProjection(
+ ReadableConfig options, DataType physicalDataType) {
+ final LogicalType physicalType = physicalDataType.getLogicalType();
+ checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
+
+ final int physicalFieldCount = LogicalTypeChecks.getFieldCount(physicalType);
+ final IntStream physicalFields = IntStream.range(0, physicalFieldCount);
+ final int[] keyProjection = createKeyFormatProjection(options, physicalDataType);
+ return physicalFields
+ .filter(pos -> IntStream.of(keyProjection).noneMatch(k -> k == pos))
+ .toArray();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Table Source Option Utils
+ // --------------------------------------------------------------------------------------------
+
+ public static List<String> getTopicListFromOptions(ReadableConfig tableOptions) {
+ return tableOptions.get(TOPICS);
+ }
+
+ public static Properties getPulsarProperties(ReadableConfig tableOptions) {
+ final Map<String, String> configs = ((Configuration) tableOptions).toMap();
+ return getPulsarProperties(configs);
+ }
+
+ public static Properties getPulsarProperties(Map<String, String> configs) {
+ return getPulsarPropertiesWithPrefix(configs, "pulsar");
+ }
+
+ public static Properties getPulsarPropertiesWithPrefix(
+ ReadableConfig tableOptions, String prefix) {
+ final Map<String, String> configs = ((Configuration) tableOptions).toMap();
+ return getPulsarPropertiesWithPrefix(configs, prefix);
+ }
+
+ public static Properties getPulsarPropertiesWithPrefix(
+ Map<String, String> configs, String prefix) {
+ final Properties pulsarProperties = new Properties();
+ configs.keySet().stream()
+ .filter(key -> key.startsWith(prefix))
+ .forEach(key -> pulsarProperties.put(key, configs.get(key)));
+ return pulsarProperties;
+ }
+
+ public static StartCursor getStartCursor(ReadableConfig tableOptions) {
+ if (tableOptions.getOptional(SOURCE_START_FROM_MESSAGE_ID).isPresent()) {
+ return parseMessageIdStartCursor(tableOptions.get(SOURCE_START_FROM_MESSAGE_ID));
+ } else if (tableOptions.getOptional(SOURCE_START_FROM_PUBLISH_TIME).isPresent()) {
+ return parsePublishTimeStartCursor(tableOptions.get(SOURCE_START_FROM_PUBLISH_TIME));
+ } else {
+ return StartCursor.earliest();
+ }
+ }
+
+ public static StopCursor getStopCursor(ReadableConfig tableOptions) {
+ if (tableOptions.getOptional(SOURCE_STOP_AT_MESSAGE_ID).isPresent()) {
+ return parseAtMessageIdStopCursor(tableOptions.get(SOURCE_STOP_AT_MESSAGE_ID));
+ } else if (tableOptions.getOptional(SOURCE_STOP_AFTER_MESSAGE_ID).isPresent()) {
+ return parseAfterMessageIdStopCursor(tableOptions.get(SOURCE_STOP_AFTER_MESSAGE_ID));
+ } else if (tableOptions.getOptional(SOURCE_STOP_AT_PUBLISH_TIME).isPresent()) {
+ return parseAtPublishTimeStopCursor(tableOptions.get(SOURCE_STOP_AT_PUBLISH_TIME));
+ } else {
+ return StopCursor.never();
+ }
+ }
+
+ public static SubscriptionType getSubscriptionType(ReadableConfig tableOptions) {
+ return tableOptions.get(SOURCE_SUBSCRIPTION_TYPE);
+ }
+
+ protected static StartCursor parseMessageIdStartCursor(String config) {
+ if (Objects.equals(config, "earliest")) {
+ return StartCursor.earliest();
+ } else if (Objects.equals(config, "latest")) {
+ return StartCursor.latest();
+ } else {
+ return StartCursor.fromMessageId(parseMessageIdString(config));
+ }
+ }
+
+ protected static StartCursor parsePublishTimeStartCursor(Long config) {
+ return StartCursor.fromPublishTime(config);
+ }
+
+ protected static StopCursor parseAtMessageIdStopCursor(String config) {
+ if (Objects.equals(config, "never")) {
+ return StopCursor.never();
+ } else if (Objects.equals(config, "latest")) {
+ return StopCursor.latest();
+ } else {
+ return StopCursor.atMessageId(parseMessageIdString(config));
+ }
+ }
+
+ protected static StopCursor parseAfterMessageIdStopCursor(String config) {
+ return StopCursor.afterMessageId(parseMessageIdString(config));
+ }
+
+ protected static StopCursor parseAtPublishTimeStopCursor(Long config) {
+ return StopCursor.atPublishTime(config);
+ }
+
+ protected static MessageIdImpl parseMessageIdString(String config) {
+ String[] tokens = config.split(":", 3);
+ checkArgument(tokens.length == 3, "MessageId format must be ledgerId:entryId:partitionId.");
+
+ try {
+ long ledgerId = Long.parseLong(tokens[0]);
+ long entryId = Long.parseLong(tokens[1]);
+ int partitionId = Integer.parseInt(tokens[2]);
+ return new MessageIdImpl(ledgerId, entryId, partitionId);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "MessageId format must be ledgerId:entryId:partitionId. "
+ + "Each id should be able to parsed to long type.");
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Table Sink Option Utils
+ // --------------------------------------------------------------------------------------------
+
+ public static TopicRouter<RowData> getTopicRouter(
+ ReadableConfig readableConfig, ClassLoader classLoader) {
+ if (!readableConfig.getOptional(SINK_CUSTOM_TOPIC_ROUTER).isPresent()) {
+ return null;
+ }
+
+ String className = readableConfig.get(SINK_CUSTOM_TOPIC_ROUTER);
+ try {
+ Class<?> clazz = Class.forName(className, true, classLoader);
+ if (!TopicRouter.class.isAssignableFrom(clazz)) {
+ throw new ValidationException(
+ String.format(
+ "Sink TopicRouter class '%s' should extend from the required class %s",
+ className, TopicRouter.class.getName()));
+ }
+ @SuppressWarnings("unchecked")
+ final TopicRouter<RowData> topicRouter =
+ InstantiationUtil.instantiate(className, TopicRouter.class, classLoader);
+
+ return topicRouter;
+ } catch (ClassNotFoundException | FlinkException e) {
+ throw new ValidationException(
+ String.format(
+ "Could not find and instantiate TopicRouter class '%s'", className),
+ e);
+ }
+ }
+
+ public static TopicRoutingMode getTopicRoutingMode(ReadableConfig readableConfig) {
+ return readableConfig.get(SINK_TOPIC_ROUTING_MODE);
+ }
+
+ public static long getMessageDelayMillis(ReadableConfig readableConfig) {
+ return readableConfig.get(SINK_MESSAGE_DELAY_INTERVAL).toMillis();
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptions.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptions.java
new file mode 100644
index 0000000..702f130
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableOptions.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.configuration.description.Description;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.apache.flink.configuration.description.TextElement.code;
+import static org.apache.flink.configuration.description.TextElement.text;
+import static org.apache.flink.table.factories.FactoryUtil.FORMAT_SUFFIX;
+
+/**
+ * Config options that is used to configure a Pulsar SQL Connector. These config options are
+ * specific to SQL Connectors only. Other runtime configurations can be found in {@link
+ * org.apache.flink.connector.pulsar.common.config.PulsarOptions}, {@link
+ * org.apache.flink.connector.pulsar.source.PulsarSourceOptions}, and {@link
+ * org.apache.flink.connector.pulsar.sink.PulsarSinkOptions}.
+ */
+@PublicEvolving
+public final class PulsarTableOptions {
+
+ private PulsarTableOptions() {}
+
+ public static final ConfigOption<List<String>> TOPICS =
+ ConfigOptions.key("topics")
+ .stringType()
+ .asList()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "Topic name(s) the table reads data from. It can be a single topic name or a list of topic names separated by a semicolon symbol (%s) like %s. "
+ + "When a list of topics configured, please ensure that all the topics are in the same schema as Flink Table need a fixed schema.",
+ code(";"), code("topic-1;topic-2"))
+ .build());
+
+ // --------------------------------------------------------------------------------------------
+ // Table Source Options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<SubscriptionType> SOURCE_SUBSCRIPTION_TYPE =
+ ConfigOptions.key("source.subscription-type")
+ .enumType(SubscriptionType.class)
+ .defaultValue(SubscriptionType.Exclusive)
+ .withDescription(
+ Description.builder()
+ .text(
+ "The [subscription type](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-subscriptions) that is supported by the [Pulsar DataStream source connector](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-source). Currently, only %s and %s subscription types are supported.",
+ code("Exclusive"), code("Shared"))
+ .build());
+
+ /**
+ * Exactly same as {@link
+ * org.apache.flink.connector.pulsar.source.PulsarSourceOptions#PULSAR_SUBSCRIPTION_NAME}.
+ * Copied because we want to have a default value for it.
+ */
+ public static final ConfigOption<String> SOURCE_SUBSCRIPTION_NAME =
+ ConfigOptions.key("source.subscription-name")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "The subscription name of the consumer that is used by the runtime [Pulsar DataStream source connector](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-source). This argument is required for constructing the consumer.")
+ .build());
+
+ public static final ConfigOption<String> SOURCE_START_FROM_MESSAGE_ID =
+ ConfigOptions.key("source.start.message-id")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "(Optional) Message id that is used to specify a consuming starting "
+ + "point for source. Use %s, %s or pass in a message id "
+ + "representation in %s, "
+ + "such as %s. This option takes precedence over "
+ + "source.start.publish-time.",
+ code("earliest"),
+ code("latest"),
+ code("ledgerId:entryId:partitionId"),
+ code("12:2:-1"))
+ .build());
+
+ public static final ConfigOption<Long> SOURCE_START_FROM_PUBLISH_TIME =
+ ConfigOptions.key("source.start.publish-time")
+ .longType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "(Optional) Publish timestamp that is used to specify a starting point for the [Pulsar DataStream source connector](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-source) to consume data. "
+ + "Option source.start.message-id takes precedence over this one.")
+ .build());
+
+ public static final ConfigOption<String> SOURCE_STOP_AT_MESSAGE_ID =
+ ConfigOptions.key("source.stop.at-message-id")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Optional message id used to specify a stop cursor for the unbounded sql "
+ + "source. Use \"never\", \"latest\" or pass in a message id "
+ + "representation in \"ledgerId:entryId:partitionId\", "
+ + "such as \"12:2:-1\"");
+
+ public static final ConfigOption<String> SOURCE_STOP_AFTER_MESSAGE_ID =
+ ConfigOptions.key("source.stop.after-message-id")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Optional message id used to specify a stop position but include the "
+ + "given message in the consuming result for the unbounded sql "
+ + "source. Pass in a message id "
+ + "representation in \"ledgerId:entryId:partitionId\", "
+ + "such as \"12:2:-1\". ");
+
+ public static final ConfigOption<Long> SOURCE_STOP_AT_PUBLISH_TIME =
+ ConfigOptions.key("source.stop.at-publish-time")
+ .longType()
+ .noDefaultValue()
+ .withDescription(
+ "Optional publish timestamp used to specify a stop cursor"
+ + " for the unbounded sql source.");
+
+ // --------------------------------------------------------------------------------------------
+ // Table Sink Options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<String> SINK_CUSTOM_TOPIC_ROUTER =
+ ConfigOptions.key("sink.custom-topic-router")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "(Optional) the custom topic router class URL that is used in the [Pulsar DataStream sink connector](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-sink). If this option is provided, the %s option will be ignored.",
+ code("sink.topic-routing-mode"))
+ .build());
+
+ public static final ConfigOption<TopicRoutingMode> SINK_TOPIC_ROUTING_MODE =
+ ConfigOptions.key("sink.topic-routing-mode")
+ .enumType(TopicRoutingMode.class)
+ .defaultValue(TopicRoutingMode.ROUND_ROBIN)
+ .withDescription(
+ Description.builder()
+ .text(
+ "(Optional) the topic routing mode. Available options are %s and %s. By default, it is set to %s. If you want to use a custom topic router, use the %s option to determine the partition for a particular message.",
+ code("round-robin"),
+ code("message-key-hash"),
+ code("round-robin"),
+ code("sink.custom-topic-router"))
+ .build());
+
+ public static final ConfigOption<Duration> SINK_MESSAGE_DELAY_INTERVAL =
+ ConfigOptions.key("sink.message-delay-interval")
+ .durationType()
+ .defaultValue(Duration.ZERO)
+ .withDescription(
+ "(Optional) the message delay delivery interval that is used in the [Pulsar DataStream sink connector](https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/datastream/pulsar/#pulsar-sink).");
+
+ // --------------------------------------------------------------------------------------------
+ // Format Options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<String> KEY_FORMAT =
+ ConfigOptions.key("key" + FORMAT_SUFFIX)
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Defines the format identifier for decoding/encoding key bytes in "
+ + "Pulsar message. The identifier is used to discover a suitable format factory.");
+
+ public static final ConfigOption<List<String>> KEY_FIELDS =
+ ConfigOptions.key("key.fields")
+ .stringType()
+ .asList()
+ .defaultValues()
+ .withDescription(
+ "An explicit list of physical columns from the table schema that are decoded/encoded from the key bytes of a Pulsar message. By default, this list is empty and thus a key is undefined.");
+
+ public static final ConfigOption<String> VALUE_FORMAT =
+ ConfigOptions.key("value" + FORMAT_SUFFIX)
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Defines the format identifier for decoding/encoding value data. "
+ + "The identifier is used to discover a suitable format factory.");
+
+ // --------------------------------------------------------------------------------------------
+ // Pulsar Options
+ // --------------------------------------------------------------------------------------------
+
+ /**
+ * Exactly same as {@link
+ * org.apache.flink.connector.pulsar.common.config.PulsarOptions#PULSAR_ADMIN_URL}. Copied here
+ * because it is a required config option and should not be included in the {@link
+ * org.apache.flink.table.factories.FactoryUtil.FactoryHelper#validateExcept(String...)} method.
+ *
+ * <p>By default all {@link org.apache.flink.connector.pulsar.common.config.PulsarOptions} are
+ * included in the validateExcept() method./p>
+ */
+ public static final ConfigOption<String> ADMIN_URL =
+ ConfigOptions.key("admin-url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text(
+ "The Pulsar service HTTP URL for the admin endpoint. For example, %s, or %s for TLS.",
+ code("http://my-broker.example.com:8080"),
+ code("https://my-broker.example.com:8443"))
+ .build());
+
+ /**
+ * Exactly same as {@link
+ * org.apache.flink.connector.pulsar.common.config.PulsarOptions#PULSAR_SERVICE_URL}. Copied
+ * here because it is a required config option and should not be included in the {@link
+ * org.apache.flink.table.factories.FactoryUtil.FactoryHelper#validateExcept(String...)} method.
+ *
+ * <p>By default all {@link org.apache.flink.connector.pulsar.common.config.PulsarOptions} are
+ * included in the validateExcept() method./p>
+ */
+ public static final ConfigOption<String> SERVICE_URL =
+ ConfigOptions.key("service-url")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ Description.builder()
+ .text("Service URL provider for Pulsar service.")
+ .linebreak()
+ .text(
+ "To connect to Pulsar using client libraries, you need to specify a Pulsar protocol URL.")
+ .linebreak()
+ .text(
+ "You can assign Pulsar protocol URLs to specific clusters and use the Pulsar scheme.")
+ .linebreak()
+ .list(
+ text(
+ "This is an example of %s: %s.",
+ code("localhost"),
+ code("pulsar://localhost:6650")),
+ text(
+ "If you have multiple brokers, the URL is as: %s",
+ code(
+ "pulsar://localhost:6550,localhost:6651,localhost:6652")),
+ text(
+ "A URL for a production Pulsar cluster is as: %s",
+ code(
+ "pulsar://pulsar.us-west.example.com:6650")),
+ text(
+ "If you use TLS authentication, the URL is as %s",
+ code(
+ "pulsar+ssl://pulsar.us-west.example.com:6651")))
+ .build());
+
+ public static final ConfigOption<Boolean> EXPLICIT =
+ ConfigOptions.key("explicit")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription("Indicate if the table is an explicit Flink table.");
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtils.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtils.java
new file mode 100644
index 0000000..1fd851b
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtils.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.types.RowKind;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.shade.com.google.common.collect.Sets;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getValueDecodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AFTER_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.pulsar.common.naming.TopicName.isValid;
+
+/** Util class for source and sink validation rules. */
+public class PulsarTableValidationUtils {
+
+ private PulsarTableValidationUtils() {}
+
+ public static void validatePrimaryKeyConstraints(
+ ObjectIdentifier tableName,
+ int[] primaryKeyIndexes,
+ FactoryUtil.TableFactoryHelper helper) {
+ final DecodingFormat<DeserializationSchema<RowData>> format =
+ getValueDecodingFormat(helper);
+ if (primaryKeyIndexes.length > 0
+ && format.getChangelogMode().containsOnly(RowKind.INSERT)) {
+ throw new ValidationException(
+ String.format(
+ "The Pulsar table '%s' with '%s' format doesn't support defining PRIMARY KEY constraint"
+ + " on the table, because it can't guarantee the semantic of primary key.",
+ tableName.asSummaryString(), format));
+ }
+ }
+
+ public static void validateTableSourceOptions(ReadableConfig tableOptions) {
+ validateTopicsConfigs(tableOptions);
+ validateStartCursorConfigs(tableOptions);
+ validateStopCursorConfigs(tableOptions);
+ validateSubscriptionTypeConfigs(tableOptions);
+ validateKeyFormatConfigs(tableOptions);
+ }
+
+ public static void validateTableSinkOptions(ReadableConfig tableOptions) {
+ validateTopicsConfigs(tableOptions);
+ validateKeyFormatConfigs(tableOptions);
+ validateSinkRoutingConfigs(tableOptions);
+ }
+
+ protected static void validateTopicsConfigs(ReadableConfig tableOptions) {
+ if (tableOptions.get(TOPICS).isEmpty()) {
+ throw new ValidationException("The topics list should not be empty.");
+ }
+
+ for (String topic : tableOptions.get(TOPICS)) {
+ if (!isValid(topic)) {
+ throw new ValidationException(
+ String.format("The topics name %s is not a valid topic name.", topic));
+ }
+ }
+ }
+
+ protected static void validateStartCursorConfigs(ReadableConfig tableOptions) {
+ if (tableOptions.getOptional(SOURCE_START_FROM_MESSAGE_ID).isPresent()
+ && tableOptions.getOptional(SOURCE_START_FROM_PUBLISH_TIME).isPresent()) {
+ throw new ValidationException(
+ String.format(
+ "Only one of %s and %s can be specified. Detected both of them",
+ SOURCE_START_FROM_MESSAGE_ID, SOURCE_START_FROM_PUBLISH_TIME));
+ }
+ }
+
+ protected static void validateStopCursorConfigs(ReadableConfig tableOptions) {
+ Set<ConfigOption<?>> conflictConfigOptions =
+ Sets.newHashSet(
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME);
+
+ long configsNums =
+ conflictConfigOptions.stream()
+ .map(tableOptions::getOptional)
+ .filter(Optional::isPresent)
+ .count();
+
+ if (configsNums > 1) {
+ throw new ValidationException(
+ String.format(
+ "Only one of %s, %s and %s can be specified. Detected more than 1 of them",
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME));
+ }
+ }
+
+ protected static void validateSubscriptionTypeConfigs(ReadableConfig tableOptions) {
+ SubscriptionType subscriptionType = tableOptions.get(SOURCE_SUBSCRIPTION_TYPE);
+
+ if (subscriptionType == SubscriptionType.Failover) {
+ throw new ValidationException(
+ String.format(
+ "%s SubscriptionType is not supported. ", SubscriptionType.Failover));
+ }
+ }
+
+ protected static void validateKeyFormatConfigs(ReadableConfig tableOptions) {
+ final Optional<String> optionalKeyFormat = tableOptions.getOptional(KEY_FORMAT);
+ final Optional<List<String>> optionalKeyFields = tableOptions.getOptional(KEY_FIELDS);
+ if (!optionalKeyFormat.isPresent() && optionalKeyFields.isPresent()) {
+ throw new ValidationException(
+ String.format(
+ "The option '%s' can only be declared if a key format is defined using '%s'.",
+ KEY_FIELDS.key(), KEY_FORMAT.key()));
+ } else if (optionalKeyFormat.isPresent()
+ && (!optionalKeyFields.isPresent() || optionalKeyFields.get().size() == 0)) {
+ throw new ValidationException(
+ String.format(
+ "A key format '%s' requires the declaration of one or more of key fields using '%s'.",
+ KEY_FORMAT.key(), KEY_FIELDS.key()));
+ }
+ }
+
+ protected static void validateSinkRoutingConfigs(ReadableConfig tableOptions) {
+ if (tableOptions.getOptional(SINK_TOPIC_ROUTING_MODE).orElse(TopicRoutingMode.ROUND_ROBIN)
+ == TopicRoutingMode.CUSTOM) {
+ throw new ValidationException(
+ String.format(
+ "Only %s and %s can be used. For %s, please use sink.custom-topic-router for"
+ + "custom topic router and do not set this config.",
+ TopicRoutingMode.ROUND_ROBIN,
+ TopicRoutingMode.MESSAGE_KEY_HASH,
+ TopicRoutingMode.CUSTOM));
+ }
+ if (tableOptions.getOptional(SINK_CUSTOM_TOPIC_ROUTER).isPresent()
+ && tableOptions.getOptional(SINK_TOPIC_ROUTING_MODE).isPresent()) {
+ throw new ValidationException(
+ String.format(
+ "Only one of %s and %s can be specified. Detected both of them",
+ SINK_CUSTOM_TOPIC_ROUTER, SINK_TOPIC_ROUTING_MODE));
+ }
+ }
+
+ protected static void validateUpsertModeKeyConstraints(
+ ReadableConfig tableOptions, int[] primaryKeyIndexes) {
+ if (!tableOptions.getOptional(KEY_FIELDS).isPresent()) {
+ throw new ValidationException(
+ "Upsert mode requires key.fields set to the primary key fields, should be set");
+ }
+
+ if (tableOptions.getOptional(KEY_FIELDS).get().size() == 0
+ || primaryKeyIndexes.length == 0) {
+ throw new ValidationException(
+ "'upsert-pulsar' require to define a PRIMARY KEY constraint. "
+ + "The PRIMARY KEY specifies which columns should be read from or write to the Pulsar message key. "
+ + "The PRIMARY KEY also defines records in the 'upsert-pulsar' table should update or delete on which keys.");
+ }
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableFactory.java
new file mode 100644
index 0000000..a972de0
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableFactory.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.pulsar.common.config.PulsarOptions;
+import org.apache.flink.connector.pulsar.sink.PulsarSinkOptions;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.source.PulsarSourceOptions;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSerializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSink;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableDeserializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableSource;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.types.RowKind;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createKeyFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createValueFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getKeyDecodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getKeyEncodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getMessageDelayMillis;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getPulsarProperties;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStartCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStopCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getSubscriptionType;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicListFromOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicRouter;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getValueDecodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getValueEncodingFormat;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.ADMIN_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.EXPLICIT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SERVICE_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_MESSAGE_DELAY_INTERVAL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AFTER_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.VALUE_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateTableSinkOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateTableSourceOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateUpsertModeKeyConstraints;
+import static org.apache.flink.table.factories.FactoryUtil.SINK_PARALLELISM;
+import static org.apache.pulsar.shade.org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+
+/**
+ * Factory for creating {@link DynamicTableSource} and {@link DynamicTableSink} for upsert-pulsar.
+ *
+ * <p>The major difference between this factory and {@link PulsarTableFactory} is the primary key
+ * options are overridden in this factory.
+ */
+public class UpsertPulsarTableFactory
+ implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+
+ public static final String IDENTIFIER = "upsert-pulsar";
+
+ public static final String DEFAULT_SUBSCRIPTION_NAME_PREFIX = "flink-upsert-pulsar-";
+
+ public static final boolean UPSERT_ENABLED = true;
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+ // Format options should be retrieved before validation.
+ final DecodingFormat<DeserializationSchema<RowData>> keyDecodingFormat =
+ getKeyDecodingFormat(helper);
+ final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat =
+ getValueDecodingFormat(helper);
+ ReadableConfig tableOptions = helper.getOptions();
+
+ // Validate configs are not conflict; each options is consumed; no unwanted configs
+ // PulsarOptions, PulsarSourceOptions and PulsarSinkOptions is not part of the validation.
+ helper.validateExcept(
+ PulsarOptions.CLIENT_CONFIG_PREFIX,
+ PulsarOptions.ADMIN_CONFIG_PREFIX,
+ PulsarSourceOptions.SOURCE_CONFIG_PREFIX,
+ PulsarSourceOptions.CONSUMER_CONFIG_PREFIX,
+ PulsarSinkOptions.PRODUCER_CONFIG_PREFIX,
+ PulsarSinkOptions.SINK_CONFIG_PREFIX);
+
+ // Retrieve configs
+ final List<String> topics = getTopicListFromOptions(tableOptions);
+ final StartCursor startCursor = getStartCursor(tableOptions);
+ final StopCursor stopCursor = getStopCursor(tableOptions);
+ final SubscriptionType subscriptionType = getSubscriptionType(tableOptions);
+
+ // Forward source configs
+ final Properties properties = getPulsarProperties(tableOptions);
+ properties.setProperty(PULSAR_ADMIN_URL.key(), tableOptions.get(ADMIN_URL));
+ properties.setProperty(PULSAR_SERVICE_URL.key(), tableOptions.get(SERVICE_URL));
+ // Set random subscriptionName if not provided
+ properties.setProperty(
+ PULSAR_SUBSCRIPTION_NAME.key(),
+ tableOptions
+ .getOptional(SOURCE_SUBSCRIPTION_NAME)
+ .orElse(DEFAULT_SUBSCRIPTION_NAME_PREFIX + randomAlphabetic(5)));
+ // Retrieve physical fields (not including computed or metadata fields),
+ // and projections and create a schema factory based on such information.
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+
+ overrideKeyFieldOptionsForUpsertMode((Configuration) tableOptions, context);
+ validateUpsertModeKeyConstraints(tableOptions, context.getPrimaryKeyIndexes());
+ validateTableSourceOptions(tableOptions);
+
+ final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
+ final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
+
+ final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory =
+ new PulsarTableDeserializationSchemaFactory(
+ physicalDataType,
+ keyDecodingFormat,
+ keyProjection,
+ valueDecodingFormat,
+ valueProjection,
+ UPSERT_ENABLED);
+
+ // Set default values for configuration not exposed to user.
+ final DecodingFormat<DeserializationSchema<RowData>> decodingFormatForMetadataPushdown =
+ valueDecodingFormat;
+ final ChangelogMode changelogMode =
+ ChangelogMode.newBuilder()
+ .addContainedKind(RowKind.UPDATE_AFTER)
+ .addContainedKind(RowKind.DELETE)
+ .build();
+
+ return new PulsarTableSource(
+ deserializationSchemaFactory,
+ decodingFormatForMetadataPushdown,
+ changelogMode,
+ topics,
+ properties,
+ startCursor,
+ stopCursor,
+ subscriptionType);
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ FactoryUtil.TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
+ // Format options should be retrieved before validation.
+ final EncodingFormat<SerializationSchema<RowData>> keyEncodingFormat =
+ getKeyEncodingFormat(helper);
+ final EncodingFormat<SerializationSchema<RowData>> valueEncodingFormat =
+ getValueEncodingFormat(helper);
+ ReadableConfig tableOptions = helper.getOptions();
+
+ // Validate configs are not conflict; each options is consumed; no unwanted configs
+ // PulsarOptions, PulsarSourceOptions and PulsarSinkOptions is not part of the validation.
+ helper.validateExcept(
+ PulsarOptions.CLIENT_CONFIG_PREFIX,
+ PulsarOptions.ADMIN_CONFIG_PREFIX,
+ PulsarSourceOptions.SOURCE_CONFIG_PREFIX,
+ PulsarSourceOptions.CONSUMER_CONFIG_PREFIX,
+ PulsarSinkOptions.PRODUCER_CONFIG_PREFIX,
+ PulsarSinkOptions.SINK_CONFIG_PREFIX);
+
+ // Retrieve configs
+ final TopicRouter<RowData> topicRouter =
+ getTopicRouter(tableOptions, context.getClassLoader());
+
+ final long messageDelayMillis = getMessageDelayMillis(tableOptions);
+ final List<String> topics = getTopicListFromOptions(tableOptions);
+
+ // Forward sink configs
+ final Properties properties = getPulsarProperties(tableOptions);
+ properties.setProperty(PULSAR_ADMIN_URL.key(), tableOptions.get(ADMIN_URL));
+ properties.setProperty(PULSAR_SERVICE_URL.key(), tableOptions.get(SERVICE_URL));
+
+ // Retrieve physical DataType (not including computed or metadata fields)
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+
+ // retrieve primary key and replace key.fields
+ overrideKeyFieldOptionsForUpsertMode((Configuration) tableOptions, context);
+ validateUpsertModeKeyConstraints(tableOptions, context.getPrimaryKeyIndexes());
+ validateTableSinkOptions(tableOptions);
+
+ final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
+ final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
+
+ final PulsarTableSerializationSchemaFactory serializationSchemaFactory =
+ new PulsarTableSerializationSchemaFactory(
+ physicalDataType,
+ keyEncodingFormat,
+ keyProjection,
+ valueEncodingFormat,
+ valueProjection,
+ UPSERT_ENABLED);
+
+ // Set default values for configuration not exposed to user.
+ final DeliveryGuarantee deliveryGuarantee = DeliveryGuarantee.AT_LEAST_ONCE;
+ final ChangelogMode changelogMode =
+ ChangelogMode.newBuilder()
+ .addContainedKind(RowKind.INSERT)
+ .addContainedKind(RowKind.UPDATE_AFTER)
+ .addContainedKind(RowKind.DELETE)
+ .build();
+ // upsert mode only supports message_key_hash
+ final TopicRoutingMode topicRoutingMode = TopicRoutingMode.MESSAGE_KEY_HASH;
+
+ return new PulsarTableSink(
+ serializationSchemaFactory,
+ changelogMode,
+ topics,
+ properties,
+ deliveryGuarantee,
+ topicRouter,
+ topicRoutingMode,
+ messageDelayMillis);
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Stream.of(TOPICS, ADMIN_URL, SERVICE_URL).collect(Collectors.toSet());
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return Stream.of(
+ FactoryUtil.FORMAT,
+ VALUE_FORMAT,
+ SOURCE_SUBSCRIPTION_NAME,
+ SOURCE_START_FROM_MESSAGE_ID,
+ SOURCE_START_FROM_PUBLISH_TIME,
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME,
+ SINK_MESSAGE_DELAY_INTERVAL,
+ SINK_PARALLELISM,
+ KEY_FORMAT,
+ EXPLICIT)
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Format and Delivery guarantee related options are not forward options.
+ *
+ * @return
+ */
+ @Override
+ public Set<ConfigOption<?>> forwardOptions() {
+ return Stream.of(
+ TOPICS,
+ ADMIN_URL,
+ SERVICE_URL,
+ SOURCE_SUBSCRIPTION_NAME,
+ SOURCE_START_FROM_MESSAGE_ID,
+ SOURCE_START_FROM_PUBLISH_TIME,
+ SOURCE_STOP_AT_MESSAGE_ID,
+ SOURCE_STOP_AFTER_MESSAGE_ID,
+ SOURCE_STOP_AT_PUBLISH_TIME,
+ SINK_MESSAGE_DELAY_INTERVAL)
+ .collect(Collectors.toSet());
+ }
+
+ private void overrideKeyFieldOptionsForUpsertMode(Configuration tableOptions, Context context) {
+ // override key fields
+ List<String> keyFields =
+ context.getCatalogTable().getResolvedSchema().getPrimaryKey().get().getColumns();
+ tableOptions.set(KEY_FIELDS, keyFields);
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchema.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchema.java
new file mode 100644
index 0000000..802b663
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchema.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.sink;
+
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.connector.pulsar.sink.config.SinkConfiguration;
+import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
+import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessage;
+import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
+import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.types.RowKind;
+
+import org.apache.pulsar.client.api.Schema;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link PulsarSerializationSchema} implementation for Pulsar SQL sink connector. It is
+ * responsible for retrieving fields from Flink row and serialize into Pulsar message key or body,
+ * and set necessary metadata fields as required.
+ */
+public class PulsarTableSerializationSchema implements PulsarSerializationSchema<RowData> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Nullable private final SerializationSchema<RowData> keySerialization;
+
+ private final RowData.FieldGetter[] keyFieldGetters;
+
+ private final SerializationSchema<RowData> valueSerialization;
+
+ private final RowData.FieldGetter[] valueFieldGetters;
+
+ private final PulsarWritableMetadata writableMetadata;
+
+ private final boolean upsertMode;
+
+ public PulsarTableSerializationSchema(
+ @Nullable SerializationSchema<RowData> keySerialization,
+ RowData.FieldGetter[] keyFieldGetters,
+ SerializationSchema<RowData> valueSerialization,
+ RowData.FieldGetter[] valueFieldGetters,
+ PulsarWritableMetadata writableMetadata,
+ boolean upsertMode) {
+ this.keySerialization = keySerialization;
+ this.keyFieldGetters = checkNotNull(keyFieldGetters);
+ this.valueSerialization = checkNotNull(valueSerialization);
+ this.valueFieldGetters = checkNotNull(valueFieldGetters);
+ this.writableMetadata = checkNotNull(writableMetadata);
+ this.upsertMode = upsertMode;
+ }
+
+ @Override
+ public void open(
+ SerializationSchema.InitializationContext initializationContext,
+ PulsarSinkContext sinkContext,
+ SinkConfiguration sinkConfiguration)
+ throws Exception {
+ if (keySerialization != null) {
+ keySerialization.open(initializationContext);
+ }
+ valueSerialization.open(initializationContext);
+ }
+
+ @Override
+ public PulsarMessage<?> serialize(RowData consumedRow, PulsarSinkContext sinkContext) {
+
+ final RowKind kind = consumedRow.getRowKind();
+ final byte[] serializedData;
+ if (upsertMode) {
+ if (kind == RowKind.DELETE || kind == RowKind.UPDATE_BEFORE) {
+ // transform the message as the tombstone message
+ serializedData = null;
+ } else {
+ // make the message to be INSERT to be compliant with the INSERT-ONLY format
+ final RowData valueRow = createProjectedRow(consumedRow, kind, valueFieldGetters);
+ valueRow.setRowKind(RowKind.INSERT);
+ serializedData = valueSerialization.serialize(valueRow);
+ }
+ } else {
+ final RowData valueRow = createProjectedRow(consumedRow, kind, valueFieldGetters);
+ serializedData = valueSerialization.serialize(valueRow);
+ }
+ PulsarMessageBuilder<byte[]> messageBuilder =
+ PulsarMessage.builder(Schema.BYTES, serializedData);
+
+ // apply metadata
+ writableMetadata.applyWritableMetadataInMessage(consumedRow, messageBuilder);
+
+ // get key row data
+ if (keySerialization != null) {
+ final RowData keyRow = createProjectedRow(consumedRow, RowKind.INSERT, keyFieldGetters);
+ final byte[] keyBytes = keySerialization.serialize(keyRow);
+ messageBuilder.keyBytes(keyBytes);
+ }
+
+ return messageBuilder.build();
+ }
+
+ private static RowData createProjectedRow(
+ RowData consumedRow, RowKind kind, RowData.FieldGetter[] fieldGetters) {
+ final int arity = fieldGetters.length;
+ final GenericRowData genericRowData = new GenericRowData(kind, arity);
+ for (int fieldPos = 0; fieldPos < arity; fieldPos++) {
+ genericRowData.setField(fieldPos, fieldGetters[fieldPos].getFieldOrNull(consumedRow));
+ }
+ return genericRowData;
+ }
+
+ /** A class to read fields from Flink row and map to a Pulsar metadata. */
+ public interface MetadataConverter extends Serializable {
+ Object read(RowData consumedRow, int pos);
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchemaFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchemaFactory.java
new file mode 100644
index 0000000..95c443b
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSerializationSchemaFactory.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.sink;
+
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
+import org.apache.flink.table.connector.Projection;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.utils.DataTypeUtils;
+
+import javax.annotation.Nullable;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Contains needed field mapping and encoding format information to construct a {@link
+ * PulsarTableSerializationSchema} instance.
+ */
+public class PulsarTableSerializationSchemaFactory {
+
+ private final DataType physicalDataType;
+
+ @Nullable private final EncodingFormat<SerializationSchema<RowData>> keyEncodingFormat;
+
+ private final int[] keyProjection;
+
+ private final EncodingFormat<SerializationSchema<RowData>> valueEncodingFormat;
+
+ private final int[] valueProjection;
+
+ /** Metadata that is appended at the end of a physical sink row. */
+ private List<String> writableMetadataKeys;
+
+ private final boolean upsertMode;
+
+ public PulsarTableSerializationSchemaFactory(
+ DataType physicalDataType,
+ @Nullable EncodingFormat<SerializationSchema<RowData>> keyEncodingFormat,
+ int[] keyProjection,
+ EncodingFormat<SerializationSchema<RowData>> valueEncodingFormat,
+ int[] valueProjection,
+ boolean upsertMode) {
+ this.physicalDataType = checkNotNull(physicalDataType);
+ this.keyEncodingFormat = keyEncodingFormat;
+ this.keyProjection = checkNotNull(keyProjection);
+ this.valueEncodingFormat = checkNotNull(valueEncodingFormat);
+ this.valueProjection = checkNotNull(valueProjection);
+ this.writableMetadataKeys = Collections.emptyList();
+ this.upsertMode = upsertMode;
+ }
+
+ public PulsarSerializationSchema<RowData> createPulsarSerializationSchema(
+ DynamicTableSink.Context context) {
+
+ final SerializationSchema<RowData> keySerialization =
+ createSerialization(context, keyEncodingFormat, keyProjection, null);
+
+ final SerializationSchema<RowData> valueSerialization =
+ createSerialization(context, valueEncodingFormat, valueProjection, null);
+ final List<LogicalType> physicalChildren = physicalDataType.getLogicalType().getChildren();
+
+ final RowData.FieldGetter[] keyFieldGetters =
+ getFieldGetters(physicalChildren, keyProjection);
+ final RowData.FieldGetter[] valueFieldGetters =
+ getFieldGetters(physicalChildren, valueProjection);
+
+ final PulsarWritableMetadata writableMetadata =
+ new PulsarWritableMetadata(writableMetadataKeys, physicalChildren.size());
+
+ return new PulsarTableSerializationSchema(
+ keySerialization,
+ keyFieldGetters,
+ valueSerialization,
+ valueFieldGetters,
+ writableMetadata,
+ upsertMode);
+ }
+
+ private @Nullable SerializationSchema<RowData> createSerialization(
+ DynamicTableSink.Context context,
+ @Nullable EncodingFormat<SerializationSchema<RowData>> format,
+ int[] projection,
+ @Nullable String prefix) {
+ if (format == null) {
+ return null;
+ }
+ DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
+ if (prefix != null) {
+ physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+ }
+ return format.createRuntimeEncoder(context, physicalFormatDataType);
+ }
+
+ private RowData.FieldGetter[] getFieldGetters(
+ List<LogicalType> physicalChildren, int[] projection) {
+ return Arrays.stream(projection)
+ .mapToObj(
+ targetField ->
+ RowData.createFieldGetter(
+ physicalChildren.get(targetField), targetField))
+ .toArray(RowData.FieldGetter[]::new);
+ }
+
+ public void setWritableMetadataKeys(List<String> writableMetadataKeys) {
+ this.writableMetadataKeys = writableMetadataKeys;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PulsarTableSerializationSchemaFactory that = (PulsarTableSerializationSchemaFactory) o;
+ return Objects.equals(physicalDataType, that.physicalDataType)
+ && Objects.equals(keyEncodingFormat, that.keyEncodingFormat)
+ && Arrays.equals(keyProjection, that.keyProjection)
+ && Objects.equals(valueEncodingFormat, that.valueEncodingFormat)
+ && Arrays.equals(valueProjection, that.valueProjection)
+ && Objects.equals(writableMetadataKeys, that.writableMetadataKeys)
+ && Objects.equals(upsertMode, that.upsertMode);
+ }
+
+ @Override
+ public int hashCode() {
+ int result =
+ Objects.hash(
+ physicalDataType,
+ keyEncodingFormat,
+ valueEncodingFormat,
+ writableMetadataKeys,
+ upsertMode);
+ result = 31 * result + Arrays.hashCode(keyProjection);
+ result = 31 * result + Arrays.hashCode(valueProjection);
+ return result;
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSink.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSink.java
new file mode 100644
index 0000000..ff0e996
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarTableSink.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.sink;
+
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.pulsar.sink.PulsarSink;
+import org.apache.flink.connector.pulsar.sink.PulsarSinkBuilder;
+import org.apache.flink.connector.pulsar.sink.writer.delayer.FixedMessageDelayer;
+import org.apache.flink.connector.pulsar.sink.writer.delayer.MessageDelayer;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.sink.writer.serializer.PulsarSerializationSchema;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkV2Provider;
+import org.apache.flink.table.connector.sink.abilities.SupportsWritingMetadata;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+
+import javax.annotation.Nullable;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.stream.Stream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** Pulsar SQL Connector sink. It supports {@link SupportsWritingMetadata}. */
+public class PulsarTableSink implements DynamicTableSink, SupportsWritingMetadata {
+
+ private final PulsarTableSerializationSchemaFactory serializationSchemaFactory;
+
+ private final ChangelogMode changelogMode;
+
+ private final List<String> topics;
+
+ private final Properties properties;
+
+ private final DeliveryGuarantee deliveryGuarantee;
+
+ @Nullable private final TopicRouter<RowData> topicRouter;
+
+ private final TopicRoutingMode topicRoutingMode;
+
+ private final long messageDelayMillis;
+
+ public PulsarTableSink(
+ PulsarTableSerializationSchemaFactory serializationSchemaFactory,
+ ChangelogMode changelogMode,
+ List<String> topics,
+ Properties properties,
+ DeliveryGuarantee deliveryGuarantee,
+ @Nullable TopicRouter<RowData> topicRouter,
+ TopicRoutingMode topicRoutingMode,
+ long messageDelayMillis) {
+ this.serializationSchemaFactory = checkNotNull(serializationSchemaFactory);
+ this.changelogMode = checkNotNull(changelogMode);
+ this.topics = checkNotNull(topics);
+ // Mutable attributes
+ this.properties = checkNotNull(properties);
+ this.deliveryGuarantee = checkNotNull(deliveryGuarantee);
+ this.topicRouter = topicRouter;
+ this.topicRoutingMode = checkNotNull(topicRoutingMode);
+ this.messageDelayMillis = messageDelayMillis;
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
+ return this.changelogMode;
+ }
+
+ @Override
+ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+
+ final PulsarSerializationSchema<RowData> pulsarSerializationSchema =
+ serializationSchemaFactory.createPulsarSerializationSchema(context);
+
+ final MessageDelayer<RowData> delayer =
+ messageDelayMillis > 0
+ ? new FixedMessageDelayer<>(messageDelayMillis)
+ : MessageDelayer.never();
+ final PulsarSinkBuilder<RowData> pulsarSinkBuilder =
+ PulsarSink.builder()
+ .setSerializationSchema(pulsarSerializationSchema)
+ .setProperties(properties)
+ .setDeliveryGuarantee(deliveryGuarantee)
+ .setTopics(topics)
+ .setTopicRoutingMode(topicRoutingMode)
+ .delaySendingMessage(delayer);
+
+ if (topicRouter != null) {
+ pulsarSinkBuilder.setTopicRouter(topicRouter);
+ }
+ return SinkV2Provider.of(pulsarSinkBuilder.build());
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Pulsar dynamic table sink";
+ }
+
+ @Override
+ public Map<String, DataType> listWritableMetadata() {
+ final Map<String, DataType> metadataMap = new LinkedHashMap<>();
+ Stream.of(PulsarWritableMetadata.WritableMetadata.values())
+ .forEachOrdered(m -> metadataMap.put(m.key, m.dataType));
+ return metadataMap;
+ }
+
+ @Override
+ public void applyWritableMetadata(List<String> metadataKeys, DataType consumedDataType) {
+ this.serializationSchemaFactory.setWritableMetadataKeys(metadataKeys);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PulsarTableSink that = (PulsarTableSink) o;
+ return Objects.equals(serializationSchemaFactory, that.serializationSchemaFactory)
+ && Objects.equals(changelogMode, that.changelogMode)
+ && Objects.equals(topics, that.topics)
+ && Objects.equals(properties, that.properties)
+ && deliveryGuarantee == that.deliveryGuarantee
+ && Objects.equals(topicRouter, that.topicRouter)
+ && topicRoutingMode == that.topicRoutingMode
+ && messageDelayMillis == that.messageDelayMillis;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ serializationSchemaFactory,
+ changelogMode,
+ topics,
+ properties,
+ deliveryGuarantee,
+ topicRouter,
+ topicRoutingMode,
+ messageDelayMillis);
+ }
+
+ @Override
+ public DynamicTableSink copy() {
+ return new PulsarTableSink(
+ serializationSchemaFactory,
+ changelogMode,
+ topics,
+ properties,
+ deliveryGuarantee,
+ topicRouter,
+ topicRoutingMode,
+ messageDelayMillis);
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarWritableMetadata.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarWritableMetadata.java
new file mode 100644
index 0000000..7cbacb9
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/sink/PulsarWritableMetadata.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.sink;
+
+import org.apache.flink.connector.pulsar.sink.writer.message.PulsarMessageBuilder;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+
+import org.bouncycastle.util.Arrays;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Stream;
+
+/** A class used to manage WritableMetadata for Pulsar SQL sink connector. */
+public class PulsarWritableMetadata implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List<String> writableMetadataKeys;
+
+ private final int physicalChildrenSize;
+
+ /**
+ * Contains the position for each value of {@link WritableMetadata} in the consumed row or -1 if
+ * this metadata key is not used.
+ */
+ private final int[] metadataPositions;
+
+ public PulsarWritableMetadata(List<String> writableMetadataKeys, int physicalChildrenSize) {
+ this.writableMetadataKeys = writableMetadataKeys;
+ this.physicalChildrenSize = physicalChildrenSize;
+ this.metadataPositions = getMetadataPositions();
+ }
+
+ public void applyWritableMetadataInMessage(
+ RowData consumedRow, PulsarMessageBuilder<byte[]> messageBuilder) {
+ Map<String, String> properties = readMetadata(consumedRow, WritableMetadata.PROPERTIES);
+ if (properties != null) {
+ messageBuilder.properties(properties);
+ }
+ final Long eventTime = readMetadata(consumedRow, WritableMetadata.EVENT_TIME);
+ if (eventTime != null && eventTime > 0) {
+ messageBuilder.eventTime(eventTime);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> T readMetadata(RowData consumedRow, WritableMetadata metadata) {
+ if (Arrays.isNullOrEmpty(metadataPositions)) {
+ return null;
+ }
+ final int pos = metadataPositions[metadata.ordinal()];
+ if (pos < 0) {
+ return null;
+ }
+ return (T) metadata.converter.read(consumedRow, pos);
+ }
+
+ private int[] getMetadataPositions() {
+ return Stream.of(WritableMetadata.values())
+ .mapToInt(
+ m -> {
+ final int pos = writableMetadataKeys.indexOf(m.key);
+ if (pos < 0) {
+ return -1;
+ }
+ return physicalChildrenSize + pos;
+ })
+ .toArray();
+ }
+
+ /** A list of writable metadata used by Pulsar SQL sink connector. */
+ public enum WritableMetadata {
+ PROPERTIES(
+ "properties",
+ // key and value of the map are nullable to make handling easier in queries
+ DataTypes.MAP(DataTypes.STRING().nullable(), DataTypes.STRING().nullable())
+ .nullable(),
+ (row, pos) -> {
+ if (row.isNullAt(pos)) {
+ return null;
+ }
+ final MapData map = row.getMap(pos);
+ final ArrayData keyArray = map.keyArray();
+ final ArrayData valueArray = map.valueArray();
+
+ final Properties properties = new Properties();
+ for (int i = 0; i < keyArray.size(); i++) {
+ if (!keyArray.isNullAt(i) && !valueArray.isNullAt(i)) {
+ final String key = keyArray.getString(i).toString();
+ final String value = valueArray.getString(i).toString();
+ properties.put(key, value);
+ }
+ }
+ return properties;
+ }),
+
+ EVENT_TIME(
+ "event_time",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).nullable(),
+ (row, pos) -> {
+ if (row.isNullAt(pos)) {
+ return null;
+ }
+ return row.getTimestamp(pos, 3).getMillisecond();
+ });
+ public final String key;
+
+ public final DataType dataType;
+
+ public final PulsarTableSerializationSchema.MetadataConverter converter;
+
+ WritableMetadata(
+ String key,
+ DataType dataType,
+ PulsarTableSerializationSchema.MetadataConverter converter) {
+ this.key = key;
+ this.dataType = dataType;
+ this.converter = converter;
+ }
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarReadableMetadata.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarReadableMetadata.java
new file mode 100644
index 0000000..7e2eeae
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarReadableMetadata.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.source;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.pulsar.client.api.Message;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Class for reading metadata fields from a Pulsar message and put in corresponding Flink row
+ * fields.
+ *
+ * <p>Contains list of readable metadata and provide util methods for metadata manipulation.
+ */
+public class PulsarReadableMetadata implements Serializable {
+
+ private static final long serialVersionUID = -4409932324481235973L;
+
+ private final List<String> connectorMetadataKeys;
+
+ private final List<MetadataConverter> metadataConverters;
+
+ public PulsarReadableMetadata(List<String> connectorMetadataKeys) {
+ this.connectorMetadataKeys = connectorMetadataKeys;
+ this.metadataConverters = initializeMetadataConverters();
+ }
+
+ private List<MetadataConverter> initializeMetadataConverters() {
+ return connectorMetadataKeys.stream()
+ .map(
+ k ->
+ Stream.of(ReadableMetadata.values())
+ .filter(rm -> rm.key.equals(k))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new))
+ .map(m -> m.converter)
+ .collect(Collectors.toList());
+ }
+
+ public void appendProducedRowWithMetadata(
+ GenericRowData producedRowData, int physicalArity, Message<?> message) {
+ for (int metadataPos = 0; metadataPos < metadataConverters.size(); metadataPos++) {
+ producedRowData.setField(
+ physicalArity + metadataPos, metadataConverters.get(metadataPos).read(message));
+ }
+ }
+
+ public int getConnectorMetadataArity() {
+ return metadataConverters.size();
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Metadata handling
+ // --------------------------------------------------------------------------------------------
+ interface MetadataConverter extends Serializable {
+ Object read(Message<?> message);
+ }
+
+ /** Lists the metadata that is readable from a Pulsar message. Used in SQL source connector. */
+ public enum ReadableMetadata {
+ TOPIC(
+ "topic",
+ DataTypes.STRING().notNull(),
+ message -> StringData.fromString(message.getTopicName())),
+
+ MESSAGE_SIZE("message_size", DataTypes.INT().notNull(), Message::size),
+
+ PRODUCER_NAME(
+ "producer_name",
+ DataTypes.STRING().notNull(),
+ message -> StringData.fromString(message.getProducerName())),
+
+ MESSAGE_ID(
+ "message_id",
+ DataTypes.BYTES().notNull(),
+ message -> message.getMessageId().toByteArray()),
+
+ SEQUENCE_ID("sequenceId", DataTypes.BIGINT().notNull(), Message::getSequenceId),
+
+ PUBLISH_TIME(
+ "publish_time",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull(),
+ message -> TimestampData.fromEpochMillis(message.getPublishTime())),
+
+ EVENT_TIME(
+ "event_time",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).notNull(),
+ message -> TimestampData.fromEpochMillis(message.getEventTime())),
+
+ PROPERTIES(
+ "properties",
+ // key and value of the map are nullable to make handling easier in queries
+ DataTypes.MAP(DataTypes.STRING().nullable(), DataTypes.STRING().nullable())
+ .notNull(),
+ message -> {
+ final Map<StringData, StringData> map = new HashMap<>();
+ for (Map.Entry<String, String> e : message.getProperties().entrySet()) {
+ map.put(
+ StringData.fromString(e.getKey()),
+ StringData.fromString(e.getValue()));
+ }
+ return new GenericMapData(map);
+ });
+
+ public final String key;
+
+ public final DataType dataType;
+
+ public final MetadataConverter converter;
+
+ ReadableMetadata(String key, DataType dataType, MetadataConverter converter) {
+ this.key = key;
+ this.dataType = dataType;
+ this.converter = converter;
+ }
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarRowDataConverter.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarRowDataConverter.java
new file mode 100644
index 0000000..52ae59e
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarRowDataConverter.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.source;
+
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.types.DeserializationException;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.apache.pulsar.client.api.Message;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Contains the projection information needed to map a Pulsar message to proper key fields, value
+ * fields and metadata fields.
+ */
+public class PulsarRowDataConverter implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final int physicalArity;
+
+ private final int[] keyProjection;
+
+ private final int[] valueProjection;
+
+ private final PulsarReadableMetadata readableMetadata;
+
+ private final boolean upsertMode;
+
+ public PulsarRowDataConverter(
+ int physicalArity,
+ int[] keyProjection,
+ int[] valueProjection,
+ PulsarReadableMetadata readableMetadata,
+ boolean upsertMode) {
+ this.physicalArity = physicalArity;
+ this.keyProjection = keyProjection;
+ this.valueProjection = valueProjection;
+ this.readableMetadata = readableMetadata;
+ this.upsertMode = upsertMode;
+ }
+
+ public void projectToProducedRowAndCollect(
+ Message<?> message,
+ List<RowData> keyRowDataList,
+ List<RowData> valueRowDataList,
+ Collector<RowData> collector) {
+ // no key defined
+ if (hasNoKeyProjection()) {
+ valueRowDataList.forEach(
+ valueRow -> emitRow(null, (GenericRowData) valueRow, collector, message));
+ } else {
+ // otherwise emit a value for each key
+ valueRowDataList.forEach(
+ valueRow ->
+ keyRowDataList.forEach(
+ keyRow ->
+ emitRow(
+ (GenericRowData) keyRow,
+ (GenericRowData) valueRow,
+ collector,
+ message)));
+ }
+ }
+
+ public void projectToRowWithNullValueRow(
+ Message<?> message, List<RowData> keyRowDataList, Collector<RowData> collector) {
+ for (RowData keyRow : keyRowDataList) {
+ emitRow((GenericRowData) keyRow, null, collector, message);
+ }
+ }
+
+ private void emitRow(
+ @Nullable GenericRowData physicalKeyRow,
+ @Nullable GenericRowData physicalValueRow,
+ Collector<RowData> collector,
+ Message<?> message) {
+
+ final RowKind rowKind;
+ if (physicalValueRow == null) {
+ if (upsertMode) {
+ rowKind = RowKind.DELETE;
+ } else {
+ throw new DeserializationException(
+ "Invalid null value received in non-upsert mode. Could not to set row kind for output record."
+ + "upsert mode is not supported yet.");
+ }
+
+ } else {
+ rowKind = physicalValueRow.getRowKind();
+ }
+
+ final GenericRowData producedRow =
+ new GenericRowData(
+ rowKind, physicalArity + readableMetadata.getConnectorMetadataArity());
+
+ for (int valuePos = 0; valuePos < valueProjection.length; valuePos++) {
+ producedRow.setField(valueProjection[valuePos], physicalValueRow.getField(valuePos));
+ }
+
+ for (int keyPos = 0; keyPos < keyProjection.length; keyPos++) {
+ assert physicalKeyRow != null;
+ producedRow.setField(keyProjection[keyPos], physicalKeyRow.getField(keyPos));
+ }
+
+ readableMetadata.appendProducedRowWithMetadata(producedRow, physicalArity, message);
+ collector.collect(producedRow);
+ }
+
+ private boolean hasNoKeyProjection() {
+ return keyProjection.length == 0;
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchema.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchema.java
new file mode 100644
index 0000000..8cc6ca9
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchema.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.source;
+
+import org.apache.flink.api.common.functions.util.ListCollector;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.connector.pulsar.source.config.SourceConfiguration;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.util.Collector;
+
+import org.apache.pulsar.client.api.Message;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A specific {@link PulsarDeserializationSchema} for {@link PulsarTableSource}.
+ *
+ * <p>Both Flink's key decoding format and value decoding format are wrapped in this class. It is
+ * responsible for getting metadata fields from a physical pulsar message body, and the final
+ * projection mapping from Pulsar message fields to Flink row.
+ *
+ * <p>After retrieving key and value bytes and convert them into a list of {@link RowData}, it then
+ * delegates metadata appending, key and value {@link RowData} combining to a {@link
+ * PulsarRowDataConverter} instance.
+ */
+public class PulsarTableDeserializationSchema implements PulsarDeserializationSchema<RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private final TypeInformation<RowData> producedTypeInfo;
+
+ @Nullable private final DeserializationSchema<RowData> keyDeserialization;
+
+ private final DeserializationSchema<RowData> valueDeserialization;
+
+ private final PulsarRowDataConverter rowDataConverter;
+
+ private final boolean upsertMode;
+
+ public PulsarTableDeserializationSchema(
+ @Nullable DeserializationSchema<RowData> keyDeserialization,
+ DeserializationSchema<RowData> valueDeserialization,
+ TypeInformation<RowData> producedTypeInfo,
+ PulsarRowDataConverter rowDataConverter,
+ boolean upsertMode) {
+ if (upsertMode) {
+ checkNotNull(keyDeserialization, "upsert mode must specify a key format");
+ }
+ this.keyDeserialization = keyDeserialization;
+ this.valueDeserialization = checkNotNull(valueDeserialization);
+ this.rowDataConverter = checkNotNull(rowDataConverter);
+ this.producedTypeInfo = checkNotNull(producedTypeInfo);
+ this.upsertMode = upsertMode;
+ }
+
+ @Override
+ public void open(PulsarInitializationContext context, SourceConfiguration configuration)
+ throws Exception {
+ if (keyDeserialization != null) {
+ keyDeserialization.open(context);
+ }
+ valueDeserialization.open(context);
+ }
+
+ @Override
+ public void deserialize(Message<byte[]> message, Collector<RowData> collector)
+ throws IOException {
+
+ // Get the key row data
+ List<RowData> keyRowData = new ArrayList<>();
+ if (keyDeserialization != null) {
+ keyDeserialization.deserialize(message.getKeyBytes(), new ListCollector<>(keyRowData));
+ }
+
+ // Get the value row data
+ List<RowData> valueRowData = new ArrayList<>();
+
+ if (upsertMode && message.getData().length == 0) {
+ rowDataConverter.projectToRowWithNullValueRow(message, keyRowData, collector);
+ return;
+ }
+
+ valueDeserialization.deserialize(message.getData(), new ListCollector<>(valueRowData));
+
+ rowDataConverter.projectToProducedRowAndCollect(
+ message, keyRowData, valueRowData, collector);
+ }
+
+ @Override
+ public TypeInformation<RowData> getProducedType() {
+ return producedTypeInfo;
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchemaFactory.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchemaFactory.java
new file mode 100644
index 0000000..32294b8
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableDeserializationSchemaFactory.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.source;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.table.connector.Projection;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.utils.DataTypeUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Contains key, value projection and format information, and use such information to create a
+ * {@link PulsarTableDeserializationSchema} instance used by runtime {@link
+ * org.apache.flink.connector.pulsar.source.PulsarSource} instance.
+ *
+ * <p>A Flink row fields has a strict order: Physical Fields (Key + value) + Format Metadata Fields
+ * Connector Metadata Fields. Physical Fields are fields come directly from Pulsar message body;
+ * Format Metadata Fields are from the extra information from the decoding format. Connector
+ * metadata fields are the ones most Pulsar messages have, such as publish time, message size and
+ * producer name.
+ *
+ * <p>In general, Physical fields + Format Metadata fields are contained in the RowData decoded
+ * using valueDecodingFormat. Only Connector Metadata fields needs to be appended to the decoded
+ * RowData. The tricky part is to put format metadata and connector metadata in the right location.
+ * This requires an explicit adjustment process.
+ *
+ * <p>For example, suppose Physical Fields (Key + value) + Format Metadata Fields + Connector
+ * Metadata Fields. has arity of 11, key projection is [0, 6], and physical value projection is [1,
+ * 2, 3, 4, 5], Then after the adjustment, key projection should be [0, 6], physical value
+ * projection should be [1, 2, 3, 4, 5] and format metadata projection should be [7], connector
+ * metadata projection should be [8, 9, 10].
+ */
+public class PulsarTableDeserializationSchemaFactory implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final DataType physicalDataType;
+
+ @Nullable private final DecodingFormat<DeserializationSchema<RowData>> keyDecodingFormat;
+
+ private final int[] keyProjection;
+
+ private final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat;
+
+ private final int[] valueProjection;
+
+ // --------------------------------------------------------------------------------------------
+ // Mutable attributes. Will be updated after the applyReadableMetadata()
+ // --------------------------------------------------------------------------------------------
+ private DataType producedDataType;
+
+ private List<String> connectorMetadataKeys;
+
+ private final boolean upsertMode;
+
+ public PulsarTableDeserializationSchemaFactory(
+ DataType physicalDataType,
+ @Nullable DecodingFormat<DeserializationSchema<RowData>> keyDecodingFormat,
+ int[] keyProjection,
+ DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat,
+ int[] valueProjection,
+ boolean upsertMode) {
+ this.physicalDataType =
+ checkNotNull(physicalDataType, "field physicalDataType must not be null.");
+ this.keyDecodingFormat = keyDecodingFormat;
+ this.keyProjection = checkNotNull(keyProjection);
+ this.valueDecodingFormat =
+ checkNotNull(valueDecodingFormat, "field valueDecodingFormat must not be null.");
+ this.valueProjection =
+ checkNotNull(valueProjection, "field valueProjection must not be null.");
+
+ this.producedDataType = physicalDataType;
+ this.connectorMetadataKeys = Collections.emptyList();
+ this.upsertMode = upsertMode;
+ }
+
+ private @Nullable DeserializationSchema<RowData> createDeserialization(
+ DynamicTableSource.Context context,
+ @Nullable DecodingFormat<DeserializationSchema<RowData>> format,
+ int[] projection,
+ @Nullable String prefix) {
+ if (format == null) {
+ return null;
+ }
+
+ DataType physicalFormatDataType = Projection.of(projection).project(this.physicalDataType);
+ if (prefix != null) {
+ physicalFormatDataType = DataTypeUtils.stripRowPrefix(physicalFormatDataType, prefix);
+ }
+ return format.createRuntimeDecoder(context, physicalFormatDataType);
+ }
+
+ public PulsarDeserializationSchema<RowData> createPulsarDeserialization(
+ ScanTableSource.ScanContext context) {
+ final DeserializationSchema<RowData> keyDeserialization =
+ createDeserialization(context, keyDecodingFormat, keyProjection, "");
+ final DeserializationSchema<RowData> valueDeserialization =
+ createDeserialization(context, valueDecodingFormat, valueProjection, "");
+
+ final TypeInformation<RowData> producedTypeInfo =
+ context.createTypeInformation(producedDataType);
+
+ final PulsarReadableMetadata readableMetadata =
+ new PulsarReadableMetadata(connectorMetadataKeys);
+
+ // Get Physical Fields (key + value) + Format Metadata arity
+ final int physicalPlusFormatMetadataArity =
+ DataType.getFieldDataTypes(producedDataType).size()
+ - readableMetadata.getConnectorMetadataArity();
+ final int[] physicalValuePlusFormatMetadataProjection =
+ adjustValueProjectionByAppendConnectorMetadata(physicalPlusFormatMetadataArity);
+
+ final PulsarRowDataConverter rowDataConverter =
+ new PulsarRowDataConverter(
+ physicalPlusFormatMetadataArity,
+ keyProjection,
+ physicalValuePlusFormatMetadataProjection,
+ readableMetadata,
+ upsertMode);
+
+ return new PulsarTableDeserializationSchema(
+ keyDeserialization,
+ valueDeserialization,
+ producedTypeInfo,
+ rowDataConverter,
+ upsertMode);
+ }
+
+ public void setProducedDataType(DataType producedDataType) {
+ this.producedDataType = producedDataType;
+ }
+
+ public void setConnectorMetadataKeys(List<String> metadataKeys) {
+ this.connectorMetadataKeys = metadataKeys;
+ }
+
+ private int[] adjustValueProjectionByAppendConnectorMetadata(
+ int physicalValuePlusFormatMetadataArity) {
+ // Concat the Physical Fields (value only) with Format metadata projection.
+ final int[] physicalValuePlusFormatMetadataProjection =
+ IntStream.concat(
+ IntStream.of(valueProjection),
+ IntStream.range(
+ keyProjection.length + valueProjection.length,
+ physicalValuePlusFormatMetadataArity))
+ .toArray();
+ return physicalValuePlusFormatMetadataProjection;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PulsarTableDeserializationSchemaFactory that = (PulsarTableDeserializationSchemaFactory) o;
+ return Objects.equals(physicalDataType, that.physicalDataType)
+ && Objects.equals(keyDecodingFormat, that.keyDecodingFormat)
+ && Arrays.equals(keyProjection, that.keyProjection)
+ && Objects.equals(valueDecodingFormat, that.valueDecodingFormat)
+ && Arrays.equals(valueProjection, that.valueProjection)
+ && Objects.equals(producedDataType, that.producedDataType)
+ && Objects.equals(connectorMetadataKeys, that.connectorMetadataKeys)
+ && Objects.equals(upsertMode, that.upsertMode);
+ }
+
+ @Override
+ public int hashCode() {
+ int result =
+ Objects.hash(
+ physicalDataType,
+ keyDecodingFormat,
+ valueDecodingFormat,
+ producedDataType,
+ connectorMetadataKeys,
+ upsertMode);
+ result = 31 * result + Arrays.hashCode(keyProjection);
+ result = 31 * result + Arrays.hashCode(valueProjection);
+ return result;
+ }
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableSource.java b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableSource.java
new file mode 100644
index 0000000..eeffc90
--- /dev/null
+++ b/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/table/source/PulsarTableSource.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.source;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.connector.pulsar.source.PulsarSource;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.SourceProvider;
+import org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link ScanTableSource} implementation for Pulsar SQL Connector. It uses a {@link
+ * SourceProvider} so it doesn't need to support {@link
+ * org.apache.flink.table.connector.source.abilities.SupportsWatermarkPushDown} interface.
+ *
+ * <p>{@link PulsarTableSource}
+ */
+public class PulsarTableSource implements ScanTableSource, SupportsReadingMetadata {
+ // --------------------------------------------------------------------------------------------
+ // Format attributes
+ // --------------------------------------------------------------------------------------------
+
+ private static final String FORMAT_METADATA_PREFIX = "value.";
+
+ private final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory;
+
+ /**
+ * Usually it is the same as the valueDecodingFormat, but use a different naming to show that it
+ * is used to list all the format metadata keys.
+ */
+ private final DecodingFormat<DeserializationSchema<RowData>> decodingFormatForReadingMetadata;
+
+ private final ChangelogMode changelogMode;
+
+ // --------------------------------------------------------------------------------------------
+ // PulsarSource needed attributes
+ // --------------------------------------------------------------------------------------------
+
+ private final List<String> topics;
+
+ private final Properties properties;
+
+ private final StartCursor startCursor;
+
+ private final StopCursor stopCursor;
+
+ private final SubscriptionType subscriptionType;
+
+ public PulsarTableSource(
+ PulsarTableDeserializationSchemaFactory deserializationSchemaFactory,
+ DecodingFormat<DeserializationSchema<RowData>> decodingFormatForReadingMetadata,
+ ChangelogMode changelogMode,
+ List<String> topics,
+ Properties properties,
+ StartCursor startCursor,
+ StopCursor stopCursor,
+ SubscriptionType subscriptionType) {
+ // Format attributes
+ this.deserializationSchemaFactory = checkNotNull(deserializationSchemaFactory);
+ this.decodingFormatForReadingMetadata = checkNotNull(decodingFormatForReadingMetadata);
+ this.changelogMode = changelogMode;
+ // DataStream connector attributes
+ this.topics = topics;
+ this.properties = checkNotNull(properties);
+ this.startCursor = checkNotNull(startCursor);
+ this.stopCursor = checkNotNull(stopCursor);
+ this.subscriptionType = checkNotNull(subscriptionType);
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode() {
+ return changelogMode;
+ }
+
+ @Override
+ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext context) {
+ PulsarDeserializationSchema<RowData> deserializationSchema =
+ deserializationSchemaFactory.createPulsarDeserialization(context);
+ PulsarSource<RowData> source =
+ PulsarSource.builder()
+ .setTopics(topics)
+ .setStartCursor(startCursor)
+ .setUnboundedStopCursor(stopCursor)
+ .setDeserializationSchema(deserializationSchema)
+ .setProperties(properties)
+ .build();
+ return SourceProvider.of(source);
+ }
+
+ /**
+ * According to convention, the order of the final row must be PHYSICAL + FORMAT METADATA +
+ * CONNECTOR METADATA where the format metadata has the highest precedence.
+ *
+ * @return
+ */
+ @Override
+ public Map<String, DataType> listReadableMetadata() {
+ final Map<String, DataType> allMetadataMap = new LinkedHashMap<>();
+
+ // add value format metadata with prefix
+ decodingFormatForReadingMetadata
+ .listReadableMetadata()
+ .forEach((key, value) -> allMetadataMap.put(FORMAT_METADATA_PREFIX + key, value));
+ // add connector metadata
+ Stream.of(PulsarReadableMetadata.ReadableMetadata.values())
+ .forEachOrdered(m -> allMetadataMap.putIfAbsent(m.key, m.dataType));
+
+ return allMetadataMap;
+ }
+
+ @Override
+ public void applyReadableMetadata(List<String> allMetadataKeys, DataType producedDataType) {
+ // separate connector and format metadata
+ final List<String> formatMetadataKeys =
+ allMetadataKeys.stream()
+ .filter(k -> k.startsWith(FORMAT_METADATA_PREFIX))
+ .collect(Collectors.toList());
+
+ final List<String> connectorMetadataKeys = new ArrayList<>(allMetadataKeys);
+ connectorMetadataKeys.removeAll(formatMetadataKeys);
+
+ // push down format metadata
+ final Map<String, DataType> formatMetadata =
+ decodingFormatForReadingMetadata.listReadableMetadata();
+ if (formatMetadata.size() > 0) {
+ final List<String> requestedFormatMetadataKeys =
+ formatMetadataKeys.stream()
+ .map(k -> k.substring(FORMAT_METADATA_PREFIX.length()))
+ .collect(Collectors.toList());
+ decodingFormatForReadingMetadata.applyReadableMetadata(requestedFormatMetadataKeys);
+ }
+
+ // update the factory attributes.
+ deserializationSchemaFactory.setConnectorMetadataKeys(connectorMetadataKeys);
+ deserializationSchemaFactory.setProducedDataType(producedDataType);
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Pulsar table source";
+ }
+
+ @Override
+ public DynamicTableSource copy() {
+ return new PulsarTableSource(
+ deserializationSchemaFactory,
+ decodingFormatForReadingMetadata,
+ changelogMode,
+ topics,
+ properties,
+ startCursor,
+ stopCursor,
+ subscriptionType);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PulsarTableSource that = (PulsarTableSource) o;
+ return Objects.equals(deserializationSchemaFactory, that.deserializationSchemaFactory)
+ && Objects.equals(
+ decodingFormatForReadingMetadata, that.decodingFormatForReadingMetadata)
+ && Objects.equals(changelogMode, that.changelogMode)
+ && Objects.equals(topics, that.topics)
+ && Objects.equals(properties, that.properties)
+ && Objects.equals(startCursor, that.startCursor)
+ && Objects.equals(stopCursor, that.stopCursor)
+ && subscriptionType == that.subscriptionType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ deserializationSchemaFactory,
+ decodingFormatForReadingMetadata,
+ changelogMode,
+ topics,
+ properties,
+ startCursor,
+ stopCursor,
+ subscriptionType);
+ }
+}
diff --git a/flink-connector-pulsar/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector-pulsar/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
new file mode 100644
index 0000000..6b2faa7
--- /dev/null
+++ b/flink-connector-pulsar/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.connector.pulsar.table.PulsarTableFactory
+org.apache.flink.connector.pulsar.table.UpsertPulsarTableFactory
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarChangelogTableITCase.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarChangelogTableITCase.java
new file mode 100644
index 0000000..59d8fbd
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarChangelogTableITCase.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.formats.json.canal.CanalJsonFormatFactory;
+import org.apache.flink.formats.json.debezium.DebeziumJsonFormatFactory;
+import org.apache.flink.formats.json.maxwell.MaxwellJsonFormatFactory;
+import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.api.config.ExecutionConfigOptions;
+import org.apache.flink.table.api.config.OptimizerConfigOptions;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.apache.pulsar.client.api.Schema;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import java.time.Duration;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.readLines;
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.waitingExpectedResults;
+
+/** IT cases for Pulsar with changelog format for Table API & SQL. */
+@ExtendWith(MiniClusterExtension.class)
+public class PulsarChangelogTableITCase extends PulsarTableTestBase {
+ @Test
+ void testPulsarDebeziumChangelogSource() throws Exception {
+ final String topic = "changelog_topic";
+ pulsar.operator().createTopic(topic, 1);
+
+ // enables MiniBatch processing to verify MiniBatch + FLIP-95, see FLINK-18769
+ TableConfig tableConf = tableEnv.getConfig();
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED, true);
+ tableConf.set(
+ ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ALLOW_LATENCY, Duration.ofSeconds(1));
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_SIZE, 5000L);
+ tableConf.set(OptimizerConfigOptions.TABLE_OPTIMIZER_AGG_PHASE_STRATEGY, "TWO_PHASE");
+
+ // ---------- Write the Debezium json into Pulsar -------------------
+ List<String> lines = readLines("debezium-data-schema-exclude.txt");
+ try {
+ writeRecordsToPulsar(topic, lines);
+ } catch (Exception e) {
+ throw new Exception("Failed to write debezium data to Pulsar.", e);
+ }
+
+ // ---------- Produce an event time stream into Pulsar -------------------
+ String sourceDDL =
+ String.format(
+ "CREATE TABLE debezium_source ("
+ // test format metadata
+ // + " origin_ts TIMESTAMP(3)
+ // METADATA FROM 'value.ingestion-timestamp' VIRTUAL," // unused
+ + " origin_table STRING METADATA FROM 'value.source.table' VIRTUAL,"
+ + " id INT NOT NULL,"
+ + " name STRING,"
+ + " description STRING,"
+ + " weight DECIMAL(10,3),"
+ + " origin_topic STRING METADATA FROM 'topic' VIRTUAL"
+ + ") WITH ("
+ + " 'connector' = '%s',"
+ + " 'topics' = '%s',"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'value.format' = '%s',"
+ + " 'pulsar.source.fetchOneMessageTime' = '100'"
+ + ")",
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ DebeziumJsonFormatFactory.IDENTIFIER);
+ String sinkDDL =
+ "CREATE TABLE debezium_sink ("
+ + " origin_topic STRING,"
+ + " origin_table STRING,"
+ + " name STRING,"
+ + " weightSum DECIMAL(10,3),"
+ + " PRIMARY KEY (name) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'sink-insert-only' = 'false'"
+ + ")";
+ tableEnv.executeSql(sourceDDL);
+ tableEnv.executeSql(sinkDDL);
+ TableResult tableResult =
+ tableEnv.executeSql(
+ "INSERT INTO debezium_sink "
+ + "SELECT FIRST_VALUE(origin_topic), FIRST_VALUE(origin_table), name, SUM(weight) "
+ + "FROM debezium_source GROUP BY name");
+ /*
+ * Debezium captures change data on the `products` table:
+ *
+ * <pre>
+ * CREATE TABLE products (
+ * id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ * name VARCHAR(255),
+ * description VARCHAR(512),
+ * weight FLOAT
+ * );
+ * ALTER TABLE products AUTO_INCREMENT = 101;
+ *
+ * INSERT INTO products
+ * VALUES (default,"scooter","Small 2-wheel scooter",3.14),
+ * (default,"car battery","12V car battery",8.1),
+ * (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
+ * (default,"hammer","12oz carpenter's hammer",0.75),
+ * (default,"hammer","14oz carpenter's hammer",0.875),
+ * (default,"hammer","16oz carpenter's hammer",1.0),
+ * (default,"rocks","box of assorted rocks",5.3),
+ * (default,"jacket","water resistent black wind breaker",0.1),
+ * (default,"spare tire","24 inch spare tire",22.2);
+ * UPDATE products SET description='18oz carpenter hammer' WHERE id=106;
+ * UPDATE products SET weight='5.1' WHERE id=107;
+ * INSERT INTO products VALUES (default,"jacket","water resistent white wind breaker",0.2);
+ * INSERT INTO products VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
+ * UPDATE products SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
+ * UPDATE products SET weight='5.17' WHERE id=111;
+ * DELETE FROM products WHERE id=111;
+ *
+ * > SELECT * FROM products;
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | id | name | description | weight |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | 101 | scooter | Small 2-wheel scooter | 3.14 |
+ * | 102 | car battery | 12V car battery | 8.1 |
+ * | 103 | 12-pack drill bits | 12-pack of drill bits with sizes ranging from #40 to #3 | 0.8 |
+ * | 104 | hammer | 12oz carpenter's hammer | 0.75 |
+ * | 105 | hammer | 14oz carpenter's hammer | 0.875 |
+ * | 106 | hammer | 18oz carpenter hammer | 1 |
+ * | 107 | rocks | box of assorted rocks | 5.1 |
+ * | 108 | jacket | water resistent black wind breaker | 0.1 |
+ * | 109 | spare tire | 24 inch spare tire | 22.2 |
+ * | 110 | jacket | new water resistent white wind breaker | 0.5 |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * </pre>
+ */
+ List<String> expected =
+ Arrays.asList(
+ "+I[persistent://public/default/changelog_topic-partition-0, products, scooter, 3.140]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, car battery, 8.100]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, 12-pack drill bits, 0.800]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, hammer, 2.625]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, rocks, 5.100]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, jacket, 0.600]",
+ "+I[persistent://public/default/changelog_topic-partition-0, products, spare tire, 22.200]");
+
+ waitingExpectedResults("debezium_sink", expected, Duration.ofSeconds(10));
+
+ // ------------- cleanup -------------------
+
+ tableResult.getJobClient().get().cancel().get(); // stop the job
+ }
+
+ @Test
+ public void testPulsarCanalChangelogSource() throws Exception {
+ final String topic = "changelog_canal";
+ pulsar.operator().createTopic(topic, 1);
+
+ // configure time zone of the Canal Json metadata "ingestion-timestamp"
+ tableEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC"));
+ // enables MiniBatch processing to verify MiniBatch + FLIP-95, see FLINK-18769
+ TableConfig tableConf = tableEnv.getConfig();
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED, true);
+ tableConf.set(
+ ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ALLOW_LATENCY, Duration.ofSeconds(1));
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_SIZE, 5000L);
+ tableConf.set(OptimizerConfigOptions.TABLE_OPTIMIZER_AGG_PHASE_STRATEGY, "TWO_PHASE");
+
+ // ---------- Write the Canal json into Pulsar -------------------
+ List<String> lines = readLines("canal-data.txt");
+ try {
+ writeRecordsToPulsar(topic, lines);
+ } catch (Exception e) {
+ throw new Exception("Failed to write canal data to Pulsar.", e);
+ }
+
+ // ---------- Produce an event time stream into Pulsar -------------------
+ String sourceDDL =
+ String.format(
+ "CREATE TABLE canal_source ("
+ // test format metadata
+ + " origin_database STRING METADATA FROM 'value.database' VIRTUAL,"
+ + " origin_table STRING METADATA FROM 'value.table' VIRTUAL,"
+ + " origin_sql_type MAP<STRING, INT> METADATA FROM 'value.sql-type' VIRTUAL,"
+ + " origin_pk_names ARRAY<STRING> METADATA FROM 'value.pk-names' VIRTUAL,"
+ + " origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,"
+ + " origin_es TIMESTAMP(3) METADATA FROM 'value.event-timestamp' VIRTUAL,"
+ + " id INT NOT NULL,"
+ + " name STRING,"
+ + " description STRING,"
+ + " weight DECIMAL(10,3),"
+ // test connector metadata
+ + " origin_topic STRING METADATA FROM 'topic' VIRTUAL,"
+ + " WATERMARK FOR origin_es AS origin_es - INTERVAL '5' SECOND"
+ + ") WITH ("
+ + " 'connector' = '%s',"
+ + " 'topics' = '%s',"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'value.format' = 'canal-json',"
+ + " 'pulsar.source.fetchOneMessageTime' = '100'"
+ + ")",
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ CanalJsonFormatFactory.IDENTIFIER);
+ String sinkDDL =
+ "CREATE TABLE canal_sink ("
+ + " origin_topic STRING,"
+ + " origin_database STRING,"
+ + " origin_table STRING,"
+ + " origin_sql_type MAP<STRING, INT>,"
+ + " origin_pk_names ARRAY<STRING>,"
+ + " origin_ts TIMESTAMP(3),"
+ + " origin_es TIMESTAMP(3),"
+ + " name STRING,"
+ + " PRIMARY KEY (name) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'sink-insert-only' = 'false'"
+ + ")";
+ tableEnv.executeSql(sourceDDL);
+ tableEnv.executeSql(sinkDDL);
+ TableResult tableResult =
+ tableEnv.executeSql(
+ "INSERT INTO canal_sink "
+ + "SELECT origin_topic, origin_database, origin_table, origin_sql_type, "
+ + "origin_pk_names, origin_ts, origin_es, name "
+ + "FROM canal_source");
+
+ /*
+ * Canal captures change data on the `products` table:
+ *
+ * <pre>
+ * CREATE TABLE products (
+ * id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ * name VARCHAR(255),
+ * description VARCHAR(512),
+ * weight FLOAT
+ * );
+ * ALTER TABLE products AUTO_INCREMENT = 101;
+ *
+ * INSERT INTO products
+ * VALUES (default,"scooter","Small 2-wheel scooter",3.14),
+ * (default,"car battery","12V car battery",8.1),
+ * (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
+ * (default,"hammer","12oz carpenter's hammer",0.75),
+ * (default,"hammer","14oz carpenter's hammer",0.875),
+ * (default,"hammer","16oz carpenter's hammer",1.0),
+ * (default,"rocks","box of assorted rocks",5.3),
+ * (default,"jacket","water resistent black wind breaker",0.1),
+ * (default,"spare tire","24 inch spare tire",22.2);
+ * UPDATE products SET description='18oz carpenter hammer' WHERE id=106;
+ * UPDATE products SET weight='5.1' WHERE id=107;
+ * INSERT INTO products VALUES (default,"jacket","water resistent white wind breaker",0.2);
+ * INSERT INTO products VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
+ * UPDATE products SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
+ * UPDATE products SET weight='5.17' WHERE id=111;
+ * DELETE FROM products WHERE id=111;
+ *
+ * > SELECT * FROM products;
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | id | name | description | weight |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | 101 | scooter | Small 2-wheel scooter | 3.14 |
+ * | 102 | car battery | 12V car battery | 8.1 |
+ * | 103 | 12-pack drill bits | 12-pack of drill bits with sizes ranging from #40 to #3 | 0.8 |
+ * | 104 | hammer | 12oz carpenter's hammer | 0.75 |
+ * | 105 | hammer | 14oz carpenter's hammer | 0.875 |
+ * | 106 | hammer | 18oz carpenter hammer | 1 |
+ * | 107 | rocks | box of assorted rocks | 5.1 |
+ * | 108 | jacket | water resistent black wind breaker | 0.1 |
+ * | 109 | spare tire | 24 inch spare tire | 22.2 |
+ * | 110 | jacket | new water resistent white wind breaker | 0.5 |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * </pre>
+ */
+
+ List<String> expected =
+ Arrays.asList(
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:38:35.477, 2020-05-13T12:38:35, 12-pack drill bits]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:38:35.477, 2020-05-13T12:38:35, spare tire]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:39:06.301, 2020-05-13T12:39:06, hammer]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:39:09.489, 2020-05-13T12:39:09, rocks]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:39:18.230, 2020-05-13T12:39:18, jacket]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:42:33.939, 2020-05-13T12:42:33, car battery]",
+ "+I[persistent://public/default/changelog_canal-partition-0, inventory, products2, {name=12, weight=7, description=12, id=4}, [id], 2020-05-13T12:42:33.939, 2020-05-13T12:42:33, scooter]");
+
+ waitingExpectedResults("canal_sink", expected, Duration.ofSeconds(10));
+
+ // ------------- cleanup -------------------
+
+ tableResult.getJobClient().get().cancel().get(); // stop the job
+ }
+
+ @Test
+ public void testPulsarMaxwellChangelogSource() throws Exception {
+ final String topic = "changelog_maxwell";
+ pulsar.operator().createTopic(topic, 1);
+
+ // configure time zone of the Maxwell Json metadata "ingestion-timestamp"
+ tableEnv.getConfig().setLocalTimeZone(ZoneId.of("UTC"));
+ // enables MiniBatch processing to verify MiniBatch + FLIP-95, see FLINK-18769
+ TableConfig tableConf = tableEnv.getConfig();
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED, true);
+ tableConf.set(
+ ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ALLOW_LATENCY, Duration.ofSeconds(1));
+ tableConf.set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_SIZE, 5000L);
+ tableConf.set(OptimizerConfigOptions.TABLE_OPTIMIZER_AGG_PHASE_STRATEGY, "TWO_PHASE");
+
+ // ---------- Write the Maxwell json into Pulsar -------------------
+ List<String> lines = readLines("maxwell-data.txt");
+ try {
+ writeRecordsToPulsar(topic, lines);
+ } catch (Exception e) {
+ throw new Exception("Failed to write maxwell data to Pulsar.", e);
+ }
+
+ // ---------- Produce an event time stream into Pulsar -------------------
+ String sourceDDL =
+ String.format(
+ "CREATE TABLE maxwell_source ("
+ // test format metadata
+ + " origin_database STRING METADATA FROM 'value.database' VIRTUAL,"
+ + " origin_table STRING METADATA FROM 'value.table' VIRTUAL,"
+ + " origin_primary_key_columns ARRAY<STRING> METADATA FROM 'value.primary-key-columns' VIRTUAL,"
+ + " origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,"
+ + " id INT NOT NULL,"
+ + " name STRING,"
+ + " description STRING,"
+ + " weight DECIMAL(10,3),"
+ // test connector metadata
+ + " origin_topic STRING METADATA FROM 'topic' VIRTUAL"
+ + ") WITH ("
+ + " 'connector' = '%s',"
+ + " 'topics' = '%s',"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'value.format' = '%s',"
+ + " 'pulsar.source.fetchOneMessageTime' = '100'"
+ + ")",
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ MaxwellJsonFormatFactory.IDENTIFIER);
+ String sinkDDL =
+ "CREATE TABLE maxwell_sink ("
+ + " origin_topic STRING,"
+ + " origin_database STRING,"
+ + " origin_table STRING,"
+ + " origin_primary_key_columns ARRAY<STRING>,"
+ + " origin_ts TIMESTAMP(3),"
+ + " name STRING,"
+ + " PRIMARY KEY (name) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'sink-insert-only' = 'false'"
+ + ")";
+ tableEnv.executeSql(sourceDDL);
+ tableEnv.executeSql(sinkDDL);
+ TableResult tableResult =
+ tableEnv.executeSql(
+ "INSERT INTO maxwell_sink "
+ + "SELECT origin_topic, origin_database, origin_table, origin_primary_key_columns, "
+ + "origin_ts, name "
+ + "FROM maxwell_source");
+
+ /*
+ * Maxwell captures change data on the `products` table:
+ *
+ * <pre>
+ * CREATE TABLE products (
+ * id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ * name VARCHAR(255),
+ * description VARCHAR(512),
+ * weight FLOAT
+ * );
+ * ALTER TABLE products AUTO_INCREMENT = 101;
+ *
+ * INSERT INTO products
+ * VALUES (default,"scooter","Small 2-wheel scooter",3.14),
+ * (default,"car battery","12V car battery",8.1),
+ * (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
+ * (default,"hammer","12oz carpenter's hammer",0.75),
+ * (default,"hammer","14oz carpenter's hammer",0.875),
+ * (default,"hammer","16oz carpenter's hammer",1.0),
+ * (default,"rocks","box of assorted rocks",5.3),
+ * (default,"jacket","water resistent black wind breaker",0.1),
+ * (default,"spare tire","24 inch spare tire",22.2);
+ * UPDATE products SET description='18oz carpenter hammer' WHERE id=106;
+ * UPDATE products SET weight='5.1' WHERE id=107;
+ * INSERT INTO products VALUES (default,"jacket","water resistent white wind breaker",0.2);
+ * INSERT INTO products VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
+ * UPDATE products SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
+ * UPDATE products SET weight='5.17' WHERE id=111;
+ * DELETE FROM products WHERE id=111;
+ *
+ * > SELECT * FROM products;
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | id | name | description | weight |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * | 101 | scooter | Small 2-wheel scooter | 3.14 |
+ * | 102 | car battery | 12V car battery | 8.1 |
+ * | 103 | 12-pack drill bits | 12-pack of drill bits with sizes ranging from #40 to #3 | 0.8 |
+ * | 104 | hammer | 12oz carpenter's hammer | 0.75 |
+ * | 105 | hammer | 14oz carpenter's hammer | 0.875 |
+ * | 106 | hammer | 18oz carpenter hammer | 1 |
+ * | 107 | rocks | box of assorted rocks | 5.1 |
+ * | 108 | jacket | water resistent black wind breaker | 0.1 |
+ * | 109 | spare tire | 24 inch spare tire | 22.2 |
+ * | 110 | jacket | new water resistent white wind breaker | 0.5 |
+ * +-----+--------------------+---------------------------------------------------------+--------+
+ * </pre>
+ */
+
+ List<String> expected =
+ Arrays.asList(
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:34:43, 12-pack drill bits]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:34:43, spare tire]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:34:53, hammer]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:34:57, rocks]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:35:06, jacket]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:35:28, car battery]",
+ "+I[persistent://public/default/changelog_maxwell-partition-0, test, product, null, 2020-08-06T03:35:28, scooter]");
+
+ waitingExpectedResults("maxwell_sink", expected, Duration.ofSeconds(10));
+
+ // ------------- cleanup -------------------
+
+ tableResult.getJobClient().get().cancel().get(); // stop the job
+ }
+
+ private void writeRecordsToPulsar(String topic, List<String> lines) throws Exception {
+ pulsar.operator().sendMessages(topic, Schema.STRING, lines);
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableFactoryTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableFactoryTest.java
new file mode 100644
index 0000000..7625c63
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableFactoryTest.java
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.connector.base.DeliveryGuarantee;
+import org.apache.flink.connector.pulsar.sink.PulsarSink;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.source.PulsarSource;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSerializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.sink.PulsarTableSink;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableDeserializationSchemaFactory;
+import org.apache.flink.connector.pulsar.table.source.PulsarTableSource;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.catalog.WatermarkSpec;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkV2Provider;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.SourceProvider;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.expressions.utils.ResolvedExpressionMock;
+import org.apache.flink.table.factories.TestFormatFactory;
+import org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext;
+import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.assertj.core.util.Lists;
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL;
+import static org.apache.flink.connector.pulsar.source.PulsarSourceOptions.PULSAR_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableFactory.UPSERT_DISABLED;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.ADMIN_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SERVICE_URL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_NAME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
+import static org.apache.flink.table.factories.FactoryUtil.FORMAT;
+import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink;
+import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * This test aims to verify that {@link PulsarTableFactory} can consume proper config options and
+ * produce expected {@link PulsarTableSource} and {@link PulsarTableSink}. It guarantees that config
+ * options is used internally by the implementation classes.
+ */
+public class PulsarTableFactoryTest {
+ private static final String TEST_TOPIC = "test-topic";
+ private static final String TEST_ADMIN_URL = "http://my-broker.example.com:8080";
+ private static final String TEST_SERVICE_URL = "pulsar://localhost:6650";
+ private static final String TEST_SUBSCRIPTION_NAME = "default-subscription";
+
+ private static final String NAME = "name";
+ private static final String COUNT = "count";
+ private static final String TIME = "time";
+ private static final String METADATA = "metadata";
+ private static final String WATERMARK_EXPRESSION = TIME + " - INTERVAL '5' SECOND";
+ private static final DataType WATERMARK_DATATYPE = DataTypes.TIMESTAMP(3);
+ private static final String COMPUTED_COLUMN_NAME = "computed-column";
+ private static final String COMPUTED_COLUMN_EXPRESSION = COUNT + " + 1.0";
+ private static final DataType COMPUTED_COLUMN_DATATYPE = DataTypes.DECIMAL(10, 3);
+
+ private static final Properties EXPECTED_PULSAR_SOURCE_PROPERTIES = new Properties();
+ private static final Properties EXPECTED_PULSAR_SINK_PROPERTIES = new Properties();
+
+ private static final String FORMAT_DELIMITER_KEY =
+ String.format("%s.%s", TestFormatFactory.IDENTIFIER, TestFormatFactory.DELIMITER.key());
+
+ private static final String FORMAT_FAIL_ON_MISSING_KEY =
+ String.format(
+ "%s.%s", TestFormatFactory.IDENTIFIER, TestFormatFactory.FAIL_ON_MISSING.key());
+
+ static {
+ EXPECTED_PULSAR_SOURCE_PROPERTIES.setProperty(PULSAR_ADMIN_URL.key(), TEST_ADMIN_URL);
+ EXPECTED_PULSAR_SOURCE_PROPERTIES.setProperty(PULSAR_SERVICE_URL.key(), TEST_SERVICE_URL);
+ EXPECTED_PULSAR_SOURCE_PROPERTIES.setProperty(
+ PULSAR_SUBSCRIPTION_NAME.key(), TEST_SUBSCRIPTION_NAME);
+
+ EXPECTED_PULSAR_SINK_PROPERTIES.setProperty(PULSAR_ADMIN_URL.key(), TEST_ADMIN_URL);
+ EXPECTED_PULSAR_SINK_PROPERTIES.setProperty(PULSAR_SERVICE_URL.key(), TEST_SERVICE_URL);
+ }
+
+ private static final ResolvedSchema SCHEMA =
+ new ResolvedSchema(
+ Arrays.asList(
+ Column.physical(NAME, DataTypes.STRING().notNull()),
+ Column.physical(COUNT, DataTypes.DECIMAL(38, 18)),
+ Column.physical(TIME, DataTypes.TIMESTAMP(3)),
+ Column.computed(
+ COMPUTED_COLUMN_NAME,
+ ResolvedExpressionMock.of(
+ COMPUTED_COLUMN_DATATYPE, COMPUTED_COLUMN_EXPRESSION))),
+ Collections.singletonList(
+ WatermarkSpec.of(
+ TIME,
+ ResolvedExpressionMock.of(
+ WATERMARK_DATATYPE, WATERMARK_EXPRESSION))),
+ null);
+
+ private static final ResolvedSchema SCHEMA_WITH_METADATA =
+ new ResolvedSchema(
+ Arrays.asList(
+ Column.physical(NAME, DataTypes.STRING()),
+ Column.physical(COUNT, DataTypes.DECIMAL(38, 18)),
+ Column.metadata(TIME, DataTypes.TIMESTAMP(3), "publish_time", false),
+ Column.metadata(
+ METADATA, DataTypes.STRING(), "value.metadata_2", false)),
+ Collections.emptyList(),
+ null);
+
+ private static final DataType SCHEMA_DATA_TYPE = SCHEMA.toPhysicalRowDataType();
+
+ @Test
+ public void testTableSource() {
+ final Map<String, String> configuration = getBasicSourceOptions();
+ final PulsarTableSource actualPulsarSource =
+ (PulsarTableSource) createTableSource(SCHEMA, configuration);
+
+ final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat =
+ new TestFormatFactory.DecodingFormatMock(",", true);
+
+ final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory =
+ new PulsarTableDeserializationSchemaFactory(
+ SCHEMA_DATA_TYPE,
+ null,
+ new int[0],
+ valueDecodingFormat,
+ new int[] {0, 1, 2},
+ UPSERT_DISABLED);
+
+ final PulsarTableSource expectedPulsarSource =
+ new PulsarTableSource(
+ deserializationSchemaFactory,
+ valueDecodingFormat,
+ valueDecodingFormat.getChangelogMode(),
+ Lists.list(TEST_TOPIC),
+ EXPECTED_PULSAR_SOURCE_PROPERTIES,
+ StartCursor.earliest(),
+ StopCursor.never(),
+ SubscriptionType.Exclusive);
+ assertThat(actualPulsarSource).isEqualTo(expectedPulsarSource);
+
+ ScanTableSource.ScanRuntimeProvider provider =
+ actualPulsarSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
+ assertPulsarSourceIsSameAsExpected(provider);
+ }
+
+ @Test
+ public void testTableSourceWithKeyValue() {
+ final Map<String, String> configuration = getSourceKeyValueOptions();
+
+ final PulsarTableSource actualPulsarSource =
+ (PulsarTableSource) createTableSource(SCHEMA, configuration);
+ // initialize stateful testing formats
+ actualPulsarSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
+
+ final TestFormatFactory.DecodingFormatMock keyDecodingFormat =
+ new TestFormatFactory.DecodingFormatMock("#", false);
+ keyDecodingFormat.producedDataType =
+ DataTypes.ROW(DataTypes.FIELD(NAME, DataTypes.STRING().notNull())).notNull();
+
+ final TestFormatFactory.DecodingFormatMock valueDecodingFormat =
+ new TestFormatFactory.DecodingFormatMock("|", false);
+ valueDecodingFormat.producedDataType =
+ DataTypes.ROW(
+ DataTypes.FIELD(COUNT, DataTypes.DECIMAL(38, 18)),
+ DataTypes.FIELD(TIME, DataTypes.TIMESTAMP(3)))
+ .notNull();
+
+ final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory =
+ new PulsarTableDeserializationSchemaFactory(
+ SCHEMA_DATA_TYPE,
+ keyDecodingFormat,
+ new int[] {0},
+ valueDecodingFormat,
+ new int[] {1, 2},
+ UPSERT_DISABLED);
+
+ final PulsarTableSource expectedPulsarSource =
+ new PulsarTableSource(
+ deserializationSchemaFactory,
+ valueDecodingFormat,
+ valueDecodingFormat.getChangelogMode(),
+ Lists.list(TEST_TOPIC),
+ EXPECTED_PULSAR_SOURCE_PROPERTIES,
+ StartCursor.earliest(),
+ StopCursor.never(),
+ SubscriptionType.Exclusive);
+
+ assertThat(actualPulsarSource).isEqualTo(expectedPulsarSource);
+ }
+
+ @Test
+ public void testTableSourceWithKeyValueAndMetadata() {
+ final Map<String, String> options = getSourceKeyValueOptions();
+ options.put("test-format.readable-metadata", "metadata_1:INT, metadata_2:STRING");
+
+ final PulsarTableSource actualPulsarSource =
+ (PulsarTableSource) createTableSource(SCHEMA_WITH_METADATA, options);
+ // initialize stateful testing formats
+ actualPulsarSource.applyReadableMetadata(
+ Arrays.asList("publish_time", "value.metadata_2"),
+ SCHEMA_WITH_METADATA.toSourceRowDataType());
+ actualPulsarSource.getScanRuntimeProvider(ScanRuntimeProviderContext.INSTANCE);
+
+ final TestFormatFactory.DecodingFormatMock expectedKeyFormat =
+ new TestFormatFactory.DecodingFormatMock(
+ "#", false, ChangelogMode.insertOnly(), Collections.emptyMap());
+ expectedKeyFormat.producedDataType =
+ DataTypes.ROW(DataTypes.FIELD(NAME, DataTypes.STRING())).notNull();
+
+ final Map<String, DataType> expectedReadableMetadata = new HashMap<>();
+ expectedReadableMetadata.put("metadata_1", DataTypes.INT());
+ expectedReadableMetadata.put("metadata_2", DataTypes.STRING());
+
+ final TestFormatFactory.DecodingFormatMock expectedValueFormat =
+ new TestFormatFactory.DecodingFormatMock(
+ "|", false, ChangelogMode.insertOnly(), expectedReadableMetadata);
+ expectedValueFormat.producedDataType =
+ DataTypes.ROW(
+ DataTypes.FIELD(COUNT, DataTypes.DECIMAL(38, 18)),
+ DataTypes.FIELD("metadata_2", DataTypes.STRING()))
+ .notNull();
+ expectedValueFormat.metadataKeys = Collections.singletonList("metadata_2");
+
+ final PulsarTableDeserializationSchemaFactory deserializationSchemaFactory =
+ new PulsarTableDeserializationSchemaFactory(
+ SCHEMA_WITH_METADATA.toPhysicalRowDataType(),
+ expectedKeyFormat,
+ new int[] {0},
+ expectedValueFormat,
+ new int[] {1},
+ UPSERT_DISABLED);
+
+ final PulsarTableSource expectedPulsarSource =
+ new PulsarTableSource(
+ deserializationSchemaFactory,
+ expectedValueFormat,
+ expectedValueFormat.getChangelogMode(),
+ Lists.list(TEST_TOPIC),
+ EXPECTED_PULSAR_SOURCE_PROPERTIES,
+ StartCursor.earliest(),
+ StopCursor.never(),
+ SubscriptionType.Exclusive);
+
+ deserializationSchemaFactory.setProducedDataType(
+ SCHEMA_WITH_METADATA.toSourceRowDataType());
+ deserializationSchemaFactory.setConnectorMetadataKeys(
+ Collections.singletonList("publish_time"));
+
+ assertThat(actualPulsarSource).isEqualTo(expectedPulsarSource);
+ }
+
+ @Test
+ public void testTableSink() {
+ final Map<String, String> modifiedOptions = getBasicSinkOptions();
+ final DynamicTableSink actualPulsarTableSink = createTableSink(SCHEMA, modifiedOptions);
+
+ final EncodingFormat<SerializationSchema<RowData>> valueEncodingFormat =
+ new TestFormatFactory.EncodingFormatMock(",");
+
+ final PulsarTableSerializationSchemaFactory serializationSchemaFactory =
+ new PulsarTableSerializationSchemaFactory(
+ SCHEMA_DATA_TYPE,
+ null,
+ new int[0],
+ valueEncodingFormat,
+ new int[] {0, 1, 2},
+ UPSERT_DISABLED);
+
+ final PulsarTableSink expectedPulsarTableSink =
+ new PulsarTableSink(
+ serializationSchemaFactory,
+ valueEncodingFormat.getChangelogMode(),
+ Lists.list(TEST_TOPIC),
+ EXPECTED_PULSAR_SINK_PROPERTIES,
+ DeliveryGuarantee.AT_LEAST_ONCE,
+ null,
+ TopicRoutingMode.ROUND_ROBIN,
+ 0);
+ assertThat(actualPulsarTableSink).isEqualTo(expectedPulsarTableSink);
+
+ DynamicTableSink.SinkRuntimeProvider provider =
+ actualPulsarTableSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
+ assertThat(provider).isInstanceOf(SinkV2Provider.class);
+ final SinkV2Provider sinkProvider = (SinkV2Provider) provider;
+ final Sink<RowData> sinkFunction = sinkProvider.createSink();
+ assertThat(sinkFunction).isInstanceOf(PulsarSink.class);
+ }
+
+ @Test
+ public void testTableSinkWithKeyValue() {
+ final Map<String, String> modifiedOptions = getSinkKeyValueOptions();
+ final PulsarTableSink actualPulsarTableSink =
+ (PulsarTableSink) createTableSink(SCHEMA, modifiedOptions);
+ // initialize stateful testing formats
+ actualPulsarTableSink.getSinkRuntimeProvider(new SinkRuntimeProviderContext(false));
+
+ final TestFormatFactory.EncodingFormatMock keyEncodingFormat =
+ new TestFormatFactory.EncodingFormatMock("#");
+ keyEncodingFormat.consumedDataType =
+ DataTypes.ROW(DataTypes.FIELD(NAME, DataTypes.STRING().notNull())).notNull();
+
+ final TestFormatFactory.EncodingFormatMock valueEncodingFormat =
+ new TestFormatFactory.EncodingFormatMock("|");
+ valueEncodingFormat.consumedDataType =
+ DataTypes.ROW(
+ DataTypes.FIELD(COUNT, DataTypes.DECIMAL(38, 18)),
+ DataTypes.FIELD(TIME, DataTypes.TIMESTAMP(3)))
+ .notNull();
+
+ final PulsarTableSerializationSchemaFactory serializationSchemaFactory =
+ new PulsarTableSerializationSchemaFactory(
+ SCHEMA_DATA_TYPE,
+ keyEncodingFormat,
+ new int[] {0},
+ valueEncodingFormat,
+ new int[] {1, 2},
+ UPSERT_DISABLED);
+
+ final PulsarTableSink expectedPulsarTableSink =
+ new PulsarTableSink(
+ serializationSchemaFactory,
+ valueEncodingFormat.getChangelogMode(),
+ Lists.list(TEST_TOPIC),
+ EXPECTED_PULSAR_SINK_PROPERTIES,
+ DeliveryGuarantee.AT_LEAST_ONCE,
+ null,
+ TopicRoutingMode.ROUND_ROBIN,
+ 0);
+ assertThat(actualPulsarTableSink).isEqualTo(expectedPulsarTableSink);
+ }
+
+ private static Map<String, String> getBasicSourceOptions() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put(CONNECTOR.key(), PulsarTableFactory.IDENTIFIER);
+ tableOptions.put(TOPICS.key(), TEST_TOPIC);
+ tableOptions.put(ADMIN_URL.key(), TEST_ADMIN_URL);
+ tableOptions.put(SERVICE_URL.key(), TEST_SERVICE_URL);
+ tableOptions.put(SOURCE_SUBSCRIPTION_NAME.key(), TEST_SUBSCRIPTION_NAME);
+ // Format options.
+ tableOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put(FORMAT_DELIMITER_KEY, ",");
+ tableOptions.put(FORMAT_FAIL_ON_MISSING_KEY, "true");
+ return tableOptions;
+ }
+
+ private static Map<String, String> getSourceKeyValueOptions() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put(CONNECTOR.key(), PulsarTableFactory.IDENTIFIER);
+ tableOptions.put(TOPICS.key(), TEST_TOPIC);
+ tableOptions.put(ADMIN_URL.key(), TEST_ADMIN_URL);
+ tableOptions.put(SERVICE_URL.key(), TEST_SERVICE_URL);
+ tableOptions.put(SOURCE_SUBSCRIPTION_NAME.key(), TEST_SUBSCRIPTION_NAME);
+ // Format options.
+ tableOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put(FORMAT_DELIMITER_KEY, "|");
+ tableOptions.put(FORMAT_FAIL_ON_MISSING_KEY, "false");
+
+ tableOptions.put(KEY_FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put("key." + FORMAT_DELIMITER_KEY, "#");
+ tableOptions.put("key." + FORMAT_FAIL_ON_MISSING_KEY, "false");
+ tableOptions.put(KEY_FIELDS.key(), NAME);
+ return tableOptions;
+ }
+
+ private static Map<String, String> getBasicSinkOptions() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put(CONNECTOR.key(), PulsarTableFactory.IDENTIFIER);
+ tableOptions.put(TOPICS.key(), TEST_TOPIC);
+ tableOptions.put(ADMIN_URL.key(), TEST_ADMIN_URL);
+ tableOptions.put(SERVICE_URL.key(), TEST_SERVICE_URL);
+ // Format options.
+ tableOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put(FORMAT_DELIMITER_KEY, ",");
+ tableOptions.put(FORMAT_FAIL_ON_MISSING_KEY, "true");
+ return tableOptions;
+ }
+
+ private static Map<String, String> getSinkKeyValueOptions() {
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put(CONNECTOR.key(), PulsarTableFactory.IDENTIFIER);
+ tableOptions.put(TOPICS.key(), TEST_TOPIC);
+ tableOptions.put(ADMIN_URL.key(), TEST_ADMIN_URL);
+ tableOptions.put(SERVICE_URL.key(), TEST_SERVICE_URL);
+ // Format options.
+ tableOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put(FORMAT_DELIMITER_KEY, "|");
+ tableOptions.put(FORMAT_FAIL_ON_MISSING_KEY, "false");
+
+ tableOptions.put(KEY_FORMAT.key(), TestFormatFactory.IDENTIFIER);
+ tableOptions.put("key." + FORMAT_DELIMITER_KEY, "#");
+ tableOptions.put("key." + FORMAT_FAIL_ON_MISSING_KEY, "false");
+ tableOptions.put(KEY_FIELDS.key(), NAME);
+ return tableOptions;
+ }
+
+ private PulsarSource<RowData> assertPulsarSourceIsSameAsExpected(
+ ScanTableSource.ScanRuntimeProvider provider) {
+ assertThat(provider).isInstanceOf(SourceProvider.class);
+ final SourceProvider sourceProvider = (SourceProvider) provider;
+ return (PulsarSource<RowData>) sourceProvider.createSource();
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableITCase.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableITCase.java
new file mode 100644
index 0000000..c1739bd
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableITCase.java
@@ -0,0 +1,678 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.connector.pulsar.table.testutils.TestingUser;
+import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+import org.apache.flink.test.util.SuccessException;
+import org.apache.flink.types.Row;
+
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.Schema;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Stream;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.collectRows;
+import static org.apache.flink.connector.pulsar.table.testutils.TestingUser.createRandomUser;
+import static org.apache.flink.util.CollectionUtil.entry;
+import static org.apache.flink.util.CollectionUtil.map;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatNoException;
+
+/**
+ * IT cases for the Pulsar table source and sink. It aims to verify runtime behaviour and certain
+ * use cases are correct and can produce/consume the desired records as user specifies.
+ */
+@ExtendWith(MiniClusterExtension.class)
+public class PulsarTableITCase extends PulsarTableTestBase {
+
+ protected static final String JSON_FORMAT = "json";
+ protected static final String AVRO_FORMAT = "avro";
+ protected static final String CSV_FORMAT = "csv";
+ protected static final String RAW_FORMAT = "raw";
+
+ @ParameterizedTest
+ @ValueSource(strings = {JSON_FORMAT, AVRO_FORMAT, CSV_FORMAT})
+ void pulsarSourceSink(String format) throws Exception {
+ // we always use a different topic name for each parameterized topic,
+ // in order to make sure the topic can be created.
+ final String topic = "test_topic_" + format + randomAlphanumeric(3);
+ createTestTopic(topic, 1);
+
+ // ---------- Produce an event time stream into Pulsar -------------------
+ String randomTableName = randomAlphabetic(5);
+ final String createTable =
+ String.format(
+ "create table %s (\n"
+ + " `computed-price` as price + 1.0,\n"
+ + " price decimal(38, 18),\n"
+ + " currency string,\n"
+ + " log_date date,\n"
+ + " log_time time(3),\n"
+ + " log_ts timestamp(3),\n"
+ + " ts as log_ts + INTERVAL '1' SECOND,\n"
+ + " watermark for ts as ts\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ randomTableName,
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createTable);
+
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "SELECT CAST(price AS DECIMAL(10, 2)), currency, "
+ + " CAST(d AS DATE), CAST(t AS TIME(0)), CAST(ts AS TIMESTAMP(3))\n"
+ + "FROM (VALUES (2.02,'Euro','2019-12-12', '00:00:01', '2019-12-12 00:00:01.001001'), \n"
+ + " (1.11,'US Dollar','2019-12-12', '00:00:02', '2019-12-12 00:00:02.002001'), \n"
+ + " (50,'Yen','2019-12-12', '00:00:03', '2019-12-12 00:00:03.004001'), \n"
+ + " (3.1,'Euro','2019-12-12', '00:00:04', '2019-12-12 00:00:04.005001'), \n"
+ + " (5.33,'US Dollar','2019-12-12', '00:00:05', '2019-12-12 00:00:05.006001'), \n"
+ + " (0,'DUMMY','2019-12-12', '00:00:10', '2019-12-12 00:00:10'))\n"
+ + " AS orders (price, currency, d, t, ts)",
+ randomTableName);
+ tableEnv.executeSql(initialValues).await();
+
+ String query =
+ String.format(
+ "SELECT\n"
+ + " CAST(TUMBLE_END(ts, INTERVAL '5' SECOND) AS VARCHAR),\n"
+ + " CAST(MAX(log_date) AS VARCHAR),\n"
+ + " CAST(MAX(log_time) AS VARCHAR),\n"
+ + " CAST(MAX(ts) AS VARCHAR),\n"
+ + " COUNT(*),\n"
+ + " CAST(MAX(price) AS DECIMAL(10, 2))\n"
+ + "FROM %s\n"
+ + "GROUP BY TUMBLE(ts, INTERVAL '5' SECOND)",
+ randomTableName);
+
+ DataStream<Row> result = tableEnv.toDataStream(tableEnv.sqlQuery(query));
+ TestingSinkFunction sink = new TestingSinkFunction(2);
+ result.addSink(sink).setParallelism(1);
+
+ try {
+ env.execute("Job_2");
+ } catch (Throwable e) {
+ if (!isCausedByJobFinished(e)) {
+ // re-throw
+ throw e;
+ }
+ }
+
+ List<String> expected =
+ Arrays.asList(
+ "+I[2019-12-12 00:00:05.000, 2019-12-12, 00:00:03, 2019-12-12 00:00:04.004, 3, 50.00]",
+ "+I[2019-12-12 00:00:10.000, 2019-12-12, 00:00:05, 2019-12-12 00:00:06.006, 2, 5.33]");
+
+ assertThat(TestingSinkFunction.rows).isEqualTo(expected);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {JSON_FORMAT, AVRO_FORMAT, CSV_FORMAT})
+ void pulsarSourceSinkWithKeyAndPartialValue(String format) throws Exception {
+ // we always use a different topic name for each parameterized topic,
+ // in order to make sure the topic can be created.
+ final String topic = "key_partial_value_topic_" + format + randomAlphanumeric(3);
+ createTestTopic(topic, 1);
+
+ // ---------- Produce an event time stream into Pulsar -------------------
+
+ // k_user_id and user_id have different data types to verify the correct mapping,
+ // fields are reordered on purpose
+ String randomTableName = randomAlphabetic(5);
+ final String createTable =
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `user_id` BIGINT,\n"
+ + " `name` STRING,\n"
+ + " `event_id` BIGINT,\n"
+ + " `payload` STRING\n"
+ + ") WITH (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s',\n"
+ + " 'key.format' = '%s',\n"
+ + " 'key.fields' = 'user_id; event_id'\n"
+ + ")",
+ randomTableName,
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format,
+ format);
+
+ tableEnv.executeSql(createTable);
+
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "VALUES\n"
+ + " (1, 'name 1', 100, 'payload 1'),\n"
+ + " (2, 'name 2', 101, 'payload 2'),\n"
+ + " (3, 'name 3', 102, 'payload 3')",
+ randomTableName);
+ tableEnv.executeSql(initialValues).await();
+
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", randomTableName)), 3);
+
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of(1L, "name 1", 100L, "payload 1"),
+ Row.of(2L, "name 2", 101L, "payload 2"),
+ Row.of(3L, "name 3", 102L, "payload 3"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {JSON_FORMAT, AVRO_FORMAT, CSV_FORMAT})
+ void pulsarSourceSinkWithMetadata(String format) throws Exception {
+ // we always use a different topic name for each parameterized topic,
+ // in order to make sure the topic can be created.
+ final String topic = "metadata_topic_" + format + randomAlphanumeric(3);
+ createTestTopic(topic, 1);
+
+ String randomTableName = randomAlphabetic(5);
+ final String createTable =
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `physical_1` STRING,\n"
+ + " `physical_2` INT,\n"
+ + " `event_time` TIMESTAMP(3) METADATA,\n"
+ + " `properties` MAP<STRING, STRING> METADATA,\n"
+ + " `physical_3` BOOLEAN\n"
+ + ") WITH (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'pulsar.producer.producerName' = 'pulsar-table-test',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ randomTableName,
+ PulsarTableFactory.IDENTIFIER,
+ topic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+ tableEnv.executeSql(createTable);
+
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "VALUES\n"
+ + " ('data 1', 1, TIMESTAMP '2022-03-24 13:12:11.123', MAP['k1', 'C0FFEE', 'k2', 'BABE01'], TRUE),\n"
+ + " ('data 2', 2, TIMESTAMP '2022-03-25 13:12:11.123', CAST(NULL AS MAP<STRING, BYTES>), FALSE),\n"
+ + " ('data 3', 3, TIMESTAMP '2022-03-26 13:12:11.123', MAP['k1', 'C0FFEE', 'k2', 'BABE01'], TRUE)",
+ randomTableName);
+ tableEnv.executeSql(initialValues).await();
+
+ // ---------- Consume stream from Pulsar -------------------
+
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", randomTableName)), 3);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of(
+ "data 1",
+ 1,
+ LocalDateTime.parse("2022-03-24T13:12:11.123"),
+ map(entry("k1", "C0FFEE"), entry("k2", "BABE01")),
+ true),
+ Row.of(
+ "data 2",
+ 2,
+ LocalDateTime.parse("2022-03-25T13:12:11.123"),
+ Collections.emptyMap(),
+ false),
+ Row.of(
+ "data 3",
+ 3,
+ LocalDateTime.parse("2022-03-26T13:12:11.123"),
+ map(entry("k1", "C0FFEE"), entry("k2", "BABE01")),
+ true));
+ }
+
+ @Test
+ void sendMessageWithPropertiesAndReadPropertiesMetadata() throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+
+ // create producer and send one message
+ String value = randomAlphabetic(5);
+ Map<String, String> properties = new HashMap<>();
+ properties.put("key1", "value1");
+ properties.put("key2", "value2");
+ try (Producer<String> producer =
+ pulsar.operator().createProducer(sourceTopic, Schema.STRING)) {
+ producer.newMessage().value(value).properties(properties).send();
+ }
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " name STRING\n,"
+ + " `properties` MAP<STRING, STRING> METADATA\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ RAW_FORMAT);
+
+ tableEnv.executeSql(createSourceTable);
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName)), 1);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ Row.of(value, map(entry("key1", "value1"), entry("key2", "value2"))));
+ }
+
+ // TODO split this into two tests
+ @ParameterizedTest
+ @MethodSource("provideSchemaData")
+ <T> void readAndSelectIntoTableUsingSimpleSchema(
+ String format, Schema<T> schema, T value, String flinkTableDataType) throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+ pulsar.operator().sendMessage(sourceTopic, schema, value);
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " `field_1` %s\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ flinkTableDataType,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSourceTable);
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName)), 1);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(value));
+
+ // insert into ... select from
+
+ final String sinkTopic = "sink_topic_" + randomAlphanumeric(3);
+ createTestTopic(sinkTopic, 1);
+ pulsar.operator().createSchema(sinkTopic, schema);
+
+ String sinkTableName = randomAlphabetic(5);
+ pulsar.operator().sendMessage(sourceTopic, schema, value);
+ final String createSinkTable =
+ String.format(
+ "create table %s (\n"
+ + " `field_1` %s\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sinkTableName,
+ flinkTableDataType,
+ PulsarTableFactory.IDENTIFIER,
+ sinkTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSinkTable);
+ tableEnv.executeSql(
+ String.format("INSERT INTO %s SELECT * FROM %s", sinkTableName, sourceTableName));
+ Message<T> sinkResult = pulsar.operator().receiveMessage(sinkTopic, schema);
+ assertThat(sinkResult.getValue()).isEqualTo(value);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideAvroBasedSchemaData")
+ void sendMessageToTopicAndReadUsingAvroBasedSchema(
+ String format, Schema<TestingUser> schema, TestingUser value) throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+ pulsar.operator().sendMessage(sourceTopic, schema, value);
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " age INT,\n"
+ + " name STRING\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSourceTable);
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName)), 1);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(value.getAge(), value.getName()));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideAvroBasedSchemaData")
+ void writeAndReadUsingAvroBasedSchema(
+ String format, Schema<TestingUser> schema, TestingUser value) throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " name STRING,\n"
+ + " age INT\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSourceTable);
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n" + "VALUES\n" + " ('%s', %s)",
+ sourceTableName, value.getName(), value.getAge());
+ tableEnv.executeSql(initialValues).await();
+
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName)), 1);
+ assertThat(result).containsExactlyInAnyOrder(Row.of(value.getName(), value.getAge()));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideAvroBasedSchemaData")
+ void selectIntoTableUsingAvroBasedSchema(
+ String format, Schema<TestingUser> schema, TestingUser value) throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+ pulsar.operator().sendMessage(sourceTopic, schema, value);
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " age INT,\n"
+ + " name STRING\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSourceTable);
+
+ final String sinkTopic = "sink_topic_" + randomAlphanumeric(3);
+ createTestTopic(sinkTopic, 1);
+ String sinkTableName = randomAlphabetic(5);
+ final String createSinkTable =
+ String.format(
+ "create table %s (\n"
+ + " age INT,\n"
+ + " name STRING\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sinkTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sinkTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format);
+
+ tableEnv.executeSql(createSinkTable);
+ tableEnv.executeSql(
+ String.format("INSERT INTO %s SELECT * FROM %s", sinkTableName, sourceTableName));
+ Message<TestingUser> sinkResult = pulsar.operator().receiveMessage(sinkTopic, schema);
+ assertThat(sinkResult.getValue()).isEqualTo(value);
+ }
+
+ @Test
+ void sendMessageToTopicAndReadUntilBoundedStopCursor() throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+ pulsar.operator().sendMessage(sourceTopic, Schema.STRING, randomAlphabetic(5));
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " name STRING\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'pulsar.source.partitionDiscoveryIntervalMs' = '-1',\n"
+ + " 'source.stop.at-message-id' = 'latest',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ RAW_FORMAT);
+
+ tableEnv.executeSql(createSourceTable);
+ JobClient jobClient =
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName))
+ .execute()
+ .getJobClient()
+ .get();
+ assertThatNoException()
+ .isThrownBy(
+ () -> {
+ JobExecutionResult result =
+ jobClient.getJobExecutionResult().get(1, TimeUnit.MINUTES);
+ });
+ }
+
+ @Test
+ void sendMessageToTopicAndReadUntilBoundedStopCursorButHasPartitionDiscovery()
+ throws Exception {
+ final String sourceTopic = "source_topic_" + randomAlphanumeric(3);
+ createTestTopic(sourceTopic, 1);
+ pulsar.operator().sendMessage(sourceTopic, Schema.STRING, randomAlphabetic(5));
+
+ String sourceTableName = randomAlphabetic(5);
+ final String createSourceTable =
+ String.format(
+ "create table %s (\n"
+ + " name STRING\n"
+ + ") with (\n"
+ + " 'connector' = '%s',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'source.stop.at-message-id' = 'latest',\n"
+ + " 'format' = '%s'\n"
+ + ")",
+ sourceTableName,
+ PulsarTableFactory.IDENTIFIER,
+ sourceTopic,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ RAW_FORMAT);
+
+ tableEnv.executeSql(createSourceTable);
+ JobClient jobClient =
+ tableEnv.sqlQuery(String.format("SELECT * FROM %s", sourceTableName))
+ .execute()
+ .getJobClient()
+ .get();
+ assertThatExceptionOfType(TimeoutException.class)
+ .isThrownBy(
+ () -> {
+ JobExecutionResult result =
+ jobClient.getJobExecutionResult().get(1, TimeUnit.MINUTES);
+ });
+ }
+
+ private static final class TestingSinkFunction implements SinkFunction<Row> {
+
+ private static final long serialVersionUID = 455430015321124493L;
+ private static List<String> rows = new ArrayList<>();
+
+ private final int expectedSize;
+
+ private TestingSinkFunction(int expectedSize) {
+ this.expectedSize = expectedSize;
+ rows.clear();
+ }
+
+ @Override
+ public void invoke(Row value, Context context) {
+ rows.add(value.toString());
+ if (rows.size() >= expectedSize) {
+ // job finish
+ throw new SuccessException();
+ }
+ }
+ }
+
+ private static boolean isCausedByJobFinished(Throwable e) {
+ if (e instanceof SuccessException) {
+ return true;
+ } else if (e.getCause() != null) {
+ return isCausedByJobFinished(e.getCause());
+ } else {
+ return false;
+ }
+ }
+
+ private static Stream<Arguments> provideSchemaData() {
+ return Stream.of(
+ Arguments.of(RAW_FORMAT, Schema.INT8, (byte) 0xa, DataTypes.TINYINT().toString()),
+ Arguments.of(
+ RAW_FORMAT, Schema.INT16, Short.MAX_VALUE, DataTypes.SMALLINT().toString()),
+ Arguments.of(
+ RAW_FORMAT, Schema.INT32, Integer.MAX_VALUE, DataTypes.INT().toString()),
+ Arguments.of(
+ RAW_FORMAT, Schema.INT64, Long.MAX_VALUE, DataTypes.BIGINT().toString()),
+ Arguments.of(
+ RAW_FORMAT, Schema.FLOAT, Float.MAX_VALUE, DataTypes.FLOAT().toString()),
+ Arguments.of(
+ RAW_FORMAT, Schema.DOUBLE, Double.MAX_VALUE, DataTypes.DOUBLE().toString()),
+ Arguments.of(RAW_FORMAT, Schema.BOOL, Boolean.TRUE, DataTypes.BOOLEAN().toString()),
+ Arguments.of(RAW_FORMAT, Schema.BYTES, new byte[1], DataTypes.BYTES().toString()),
+ Arguments.of(
+ RAW_FORMAT,
+ Schema.STRING,
+ "this is a string",
+ DataTypes.STRING().toString()));
+ }
+
+ private static Stream<Arguments> provideAvroBasedSchemaData() {
+ return Stream.of(
+ Arguments.of(JSON_FORMAT, Schema.JSON(TestingUser.class), createRandomUser()),
+ Arguments.of(AVRO_FORMAT, Schema.AVRO(TestingUser.class), createRandomUser()));
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtilsTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtilsTest.java
new file mode 100644
index 0000000..825281c
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionUtilsTest.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRoutingMode;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.StopCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.start.MessageIdStartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.start.TimestampStartCursor;
+import org.apache.flink.connector.pulsar.source.enumerator.cursor.stop.PublishTimestampStopCursor;
+import org.apache.flink.connector.pulsar.table.testutils.MockTopicRouter;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.utils.FactoryMocks;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.client.impl.MessageIdImpl;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_STATS_INTERVAL_SECONDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createKeyFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.createValueFormatProjection;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getMessageDelayMillis;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getPulsarProperties;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStartCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getStopCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getSubscriptionType;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicListFromOptions;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicRouter;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.getTopicRoutingMode;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.parseAfterMessageIdStopCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.parseAtMessageIdStopCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.parseMessageIdStartCursor;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptionUtils.parseMessageIdString;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_MESSAGE_DELAY_INTERVAL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.ROW;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatNoException;
+
+/**
+ * Unit test for {@link PulsarTableOptionUtils}. Tests each method and different inputs. Some tests
+ * have overlapping semantics with {@link PulsarTableOptionsTest} and {@link
+ * PulsarTableValidationUtilsTest}, but they cover different aspects of the validation, so all of
+ * them should be kept.
+ */
+public class PulsarTableOptionUtilsTest {
+ // --------------------------------------------------------------------------------------------
+ // Format and Projection Test
+ // --------------------------------------------------------------------------------------------
+ @Test
+ void formatProjection() {
+ final DataType dataType =
+ DataTypes.ROW(
+ FIELD("id", INT()),
+ FIELD("name", STRING()),
+ FIELD("age", INT()),
+ FIELD("address", STRING()));
+
+ final Map<String, String> options = createTestOptions();
+ options.put("key.fields", "address; name");
+
+ final Configuration config = Configuration.fromMap(options);
+
+ assertThat(createKeyFormatProjection(config, dataType)).containsExactly(3, 1);
+ assertThat(createValueFormatProjection(config, dataType)).containsExactly(0, 2);
+ }
+
+ @Test
+ void invalidKeyFormatFieldProjection() {
+ final DataType dataType = ROW(FIELD("id", INT()), FIELD("name", STRING()));
+ final Map<String, String> options = createTestOptions();
+ options.put("key.fields", "non_existing");
+
+ final Configuration config = Configuration.fromMap(options);
+
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> createKeyFormatProjection(config, dataType))
+ .withMessage(
+ String.format(
+ "Could not find the field '%s' in the table schema for usage in the key format. "
+ + "A key field must be a regular, physical column. "
+ + "The following columns can be selected in the '%s' option: [id, name]",
+ "non_existing", KEY_FIELDS.key()));
+ }
+
+ static Map<String, String> createTestOptions() {
+ final Map<String, String> options = new HashMap<>();
+ options.put("key.format", "test-format");
+ options.put("key.test-format.delimiter", ",");
+ options.put("value.format", "test-format");
+ options.put("value.test-format.delimiter", "|");
+ options.put("value.test-format.fail-on-missing", "true");
+ return options;
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Table Source Option Utils Test
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ void topicsList() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(TOPICS.key(), ";");
+ List<String> topicList = getTopicListFromOptions(Configuration.fromMap(options));
+ assertThat(topicList).isEmpty();
+
+ options.put(TOPICS.key(), "topic1;");
+ topicList = getTopicListFromOptions(Configuration.fromMap(options));
+ assertThat(topicList).hasSize(1);
+
+ options.put(TOPICS.key(), "topic1;topic2");
+ topicList = getTopicListFromOptions(Configuration.fromMap(options));
+ assertThat(topicList).hasSize(2);
+
+ options.put(TOPICS.key(), "");
+ topicList = getTopicListFromOptions(Configuration.fromMap(options));
+ assertThat(topicList).isEmpty();
+ }
+
+ @Test
+ void pulsarProperties() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(PULSAR_STATS_INTERVAL_SECONDS.key(), "30");
+ Properties properties = getPulsarProperties(Configuration.fromMap(options));
+ assertThat(properties.getProperty(PULSAR_STATS_INTERVAL_SECONDS.key())).isEqualTo("30");
+ }
+
+ @Test
+ void startCursor() {
+ // TDOO Use isEqualTo() to assert; need equals() method
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "earliest");
+ StartCursor startCursor = getStartCursor(Configuration.fromMap(options));
+ assertThat(startCursor).isInstanceOf(MessageIdStartCursor.class);
+
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "latest");
+ startCursor = getStartCursor(Configuration.fromMap(options));
+ assertThat(startCursor).isInstanceOf(MessageIdStartCursor.class);
+
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "0:0:-1");
+ startCursor = getStartCursor(Configuration.fromMap(options));
+ assertThat(startCursor).isInstanceOf(MessageIdStartCursor.class);
+
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "other");
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> getStartCursor(Configuration.fromMap(options)))
+ .withMessage("MessageId format must be ledgerId:entryId:partitionId.");
+
+ options.remove(SOURCE_START_FROM_MESSAGE_ID.key());
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "123545");
+ startCursor = getStartCursor(Configuration.fromMap(options));
+ assertThat(startCursor).isInstanceOf(TimestampStartCursor.class);
+
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "123545L");
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> getStartCursor(Configuration.fromMap(options)))
+ .withMessage(
+ "Could not parse value '123545L' for key 'source.start.publish-time'.");
+ }
+
+ @Test
+ void subscriptionType() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Shared");
+ SubscriptionType subscriptionType = getSubscriptionType(Configuration.fromMap(options));
+ assertThat(subscriptionType).isEqualTo(SubscriptionType.Shared);
+
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Exclusive");
+ subscriptionType = getSubscriptionType(Configuration.fromMap(options));
+ assertThat(subscriptionType).isEqualTo(SubscriptionType.Exclusive);
+ }
+
+ @Test
+ void canParseMessageIdEarliestOrLatestStartCursor() {
+ String earliest = "earliest";
+ StartCursor startCursor = parseMessageIdStartCursor(earliest);
+ assertThat(startCursor).isEqualTo(StartCursor.earliest());
+
+ String latest = "latest";
+ startCursor = parseMessageIdStartCursor(latest);
+ assertThat(startCursor).isEqualTo(StartCursor.latest());
+
+ String precise = "0:0:100";
+ startCursor = parseMessageIdStartCursor(precise);
+ assertThat(startCursor).isEqualTo(StartCursor.fromMessageId(new MessageIdImpl(0, 0, 100)));
+ }
+
+ @Test
+ void publishTimeStartCursor() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "12345");
+ StartCursor startCursor = getStartCursor(Configuration.fromMap(options));
+ assertThat(startCursor).isInstanceOf(TimestampStartCursor.class);
+
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "12345L");
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> getStartCursor(Configuration.fromMap(options)))
+ .withMessage("Could not parse value '12345L' for key 'source.start.publish-time'.");
+ }
+
+ @Test
+ void canParseMessageIdNeverOrLatestStopCursor() {
+ String never = "never";
+ StopCursor stopCursor = parseAtMessageIdStopCursor(never);
+ assertThat(stopCursor).isEqualTo(StopCursor.never());
+
+ String latest = "latest";
+ stopCursor = parseAtMessageIdStopCursor(latest);
+ assertThat(stopCursor).isEqualTo(StopCursor.latest());
+
+ String precise = "0:0:100";
+ stopCursor = parseAtMessageIdStopCursor(precise);
+ assertThat(stopCursor).isEqualTo(StopCursor.atMessageId(new MessageIdImpl(0, 0, 100)));
+
+ stopCursor = parseAfterMessageIdStopCursor(precise);
+ assertThat(stopCursor).isEqualTo(StopCursor.afterMessageId(new MessageIdImpl(0, 0, 100)));
+ }
+
+ @Test
+ void publishTimeStopCursor() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SOURCE_STOP_AT_PUBLISH_TIME.key(), "12345");
+ StopCursor stopCursor = getStopCursor(Configuration.fromMap(options));
+ assertThat(stopCursor).isInstanceOf(PublishTimestampStopCursor.class);
+
+ options.put(SOURCE_STOP_AT_PUBLISH_TIME.key(), "12345L");
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> getStopCursor(Configuration.fromMap(options)))
+ .withMessage(
+ "Could not parse value '12345L' for key 'source.stop.at-publish-time'.");
+ }
+
+ @Test
+ void canParseMessageIdUsingMessageIdImpl() {
+ final String invalidFormatMessage =
+ "MessageId format must be ledgerId:entryId:partitionId.";
+ final String invalidNumberMessage =
+ "MessageId format must be ledgerId:entryId:partitionId. Each id should be able to parsed to long type.";
+ String precise = "0:0:100";
+ assertThatNoException().isThrownBy(() -> parseMessageIdString(precise));
+
+ String empty = "";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(empty))
+ .withMessage(invalidFormatMessage);
+
+ String noSemicolon = "0";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(noSemicolon))
+ .withMessage(invalidFormatMessage);
+
+ String oneSemiColon = "0:";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(oneSemiColon))
+ .withMessage(invalidFormatMessage);
+
+ String oneSemiColonComplete = "0:0";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(oneSemiColonComplete))
+ .withMessage(invalidFormatMessage);
+
+ String twoSemiColon = "0:0:";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(twoSemiColon))
+ .withMessage(invalidNumberMessage);
+
+ String twoSemiColonComplete = "0:0:0";
+ assertThatNoException().isThrownBy(() -> parseMessageIdString(twoSemiColonComplete));
+
+ String threeSemicolon = "0:0:0:";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(threeSemicolon))
+ .withMessage(invalidNumberMessage);
+
+ String threeSemicolonComplete = "0:0:0:0";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(threeSemicolonComplete))
+ .withMessage(invalidNumberMessage);
+
+ String invalidNumber = "0:0:adf";
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> parseMessageIdString(invalidNumber))
+ .withMessage(invalidNumberMessage);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Table Sink Option Utils Test
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ void topicRouter() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(
+ SINK_CUSTOM_TOPIC_ROUTER.key(),
+ "org.apache.flink.connector.pulsar.table.testutils.MockTopicRouter");
+ TopicRouter<RowData> topicRouter =
+ getTopicRouter(Configuration.fromMap(options), FactoryMocks.class.getClassLoader());
+ assertThat(topicRouter).isInstanceOf(MockTopicRouter.class);
+
+ options.put(
+ SINK_CUSTOM_TOPIC_ROUTER.key(),
+ "org.apache.flink.connector.pulsar.table.PulsarTableOptionUtilsTest");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(
+ () ->
+ getTopicRouter(
+ Configuration.fromMap(options),
+ FactoryMocks.class.getClassLoader()))
+ .withMessage(
+ String.format(
+ "Sink TopicRouter class '%s' should extend from the required class %s",
+ PulsarTableOptionUtilsTest.class.getName(),
+ TopicRouter.class.getName()));
+
+ options.put(
+ SINK_CUSTOM_TOPIC_ROUTER.key(),
+ "org.apache.flink.connector.pulsar.table.testutils.NonExistMockTopicRouter");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(
+ () ->
+ getTopicRouter(
+ Configuration.fromMap(options),
+ FactoryMocks.class.getClassLoader()))
+ .withMessage(
+ String.format(
+ "Could not find and instantiate TopicRouter class '%s'",
+ "org.apache.flink.connector.pulsar.table.testutils.NonExistMockTopicRouter"));
+ }
+
+ @Test
+ void topicRoutingMode() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SINK_TOPIC_ROUTING_MODE.key(), "round-robin");
+ TopicRoutingMode topicRoutingMode = getTopicRoutingMode(Configuration.fromMap(options));
+ assertThat(topicRoutingMode).isEqualTo(TopicRoutingMode.ROUND_ROBIN);
+
+ options.put(SINK_TOPIC_ROUTING_MODE.key(), "message-key-hash");
+ topicRoutingMode = getTopicRoutingMode(Configuration.fromMap(options));
+ assertThat(topicRoutingMode).isEqualTo(TopicRoutingMode.MESSAGE_KEY_HASH);
+ }
+
+ @Test
+ void messageDelayMillis() {
+ final Map<String, String> options = createDefaultOptions();
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "10 s");
+ long messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofSeconds(10).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "10s");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofSeconds(10).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "1000ms");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofMillis(1000).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "1 d");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofDays(1).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "1 H");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofHours(1).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "1 min");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofMinutes(1).toMillis());
+
+ options.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "1m");
+ messageDelayMillis = getMessageDelayMillis(Configuration.fromMap(options));
+ assertThat(messageDelayMillis).isEqualTo(Duration.ofMinutes(1).toMillis());
+ }
+
+ private Map<String, String> createDefaultOptions() {
+ Map<String, String> optionMap = new HashMap<>();
+ return optionMap;
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionsTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionsTest.java
new file mode 100644
index 0000000..e01289c
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableOptionsTest.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.connector.pulsar.table.testutils.MockPulsarAuthentication;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.factories.FactoryUtil;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PARAM_MAP;
+import static org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_AUTH_PLUGIN_CLASS_NAME;
+import static org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_MESSAGE_DELAY_INTERVAL;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_STOP_AT_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.table.factories.TestDynamicTableFactory.VALUE_FORMAT;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatNoException;
+
+/**
+ * Test config options for Pulsar SQL connector. This test aims to verify legal combination of
+ * config options will be accepted and do not cause runtime exceptions (but cannot guarantee they
+ * are taking effect), and illegal combinations of config options will be rejected early.
+ */
+public class PulsarTableOptionsTest extends PulsarTableTestBase {
+ @Test
+ void noTopicsSpecified() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithFormat();
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName,
+ new ValidationException(
+ "One or more required options are missing.\n\n"
+ + "Missing required options are:\n\n"
+ + "topics"));
+ }
+
+ @Test
+ void invalidEmptyTopics() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithFormat();
+ testConfigs.put(TOPICS.key(), "");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName, new ValidationException("The topics list should not be empty."));
+ }
+
+ @Test
+ void topicsWithSemicolon() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithFormat();
+ testConfigs.put(TOPICS.key(), topicName + ";");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void invalidTopicName() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithFormat();
+ String invalidTopicName = "persistent://tenant/no-topic";
+ testConfigs.put(TOPICS.key(), invalidTopicName);
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName,
+ new ValidationException(
+ String.format(
+ "The topics name %s is not a valid topic name.",
+ invalidTopicName)));
+ }
+
+ @Test
+ void topicsList() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithFormat();
+ testConfigs.put(
+ TOPICS.key(),
+ topicNameWithPartition(topicName, 0) + ";" + topicNameWithPartition(topicName, 1));
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void usingFormat() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopic(topicName);
+ testConfigs.put(FactoryUtil.FORMAT.key(), "json");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void usingValueFormat() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopic(topicName);
+ testConfigs.put(VALUE_FORMAT.key(), "json");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void usingValueFormatAndFormatOptions() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopic(topicName);
+ testConfigs.put(VALUE_FORMAT.key(), "json");
+ testConfigs.put("value.json.fail-on-missing-field", "false");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void subscriptionType() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+ testConfigs.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Exclusive");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void invalidUnsupportedSubscriptionType() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+ testConfigs.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Failover");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectException(
+ topicName, new ValidationException("Failover SubscriptionType is not supported. "));
+ }
+
+ @Test
+ void invalidNonExistSubscriptionType() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+ testConfigs.put(SOURCE_SUBSCRIPTION_TYPE.key(), "random-subscription-type");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName,
+ new ValidationException("Invalid value for option 'source.subscription-type'."));
+ }
+
+ @Test
+ void messageIdStartCursorEarliest() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_MESSAGE_ID.key(), "earliest");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void messageIdStartCursorLatest() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_MESSAGE_ID.key(), "latest");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void messageIdStartCursorExact() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_MESSAGE_ID.key(), "0:0:-1");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void invalidMessageIdStartCursorEmptyId() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_MESSAGE_ID.key(), "0:0:");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectException(
+ topicName,
+ new IllegalArgumentException(
+ "MessageId format must be ledgerId:entryId:partitionId. "
+ + "Each id should be able to parsed to long type."));
+ }
+
+ @Test
+ void invalidMessageIdStartCursorIncomplete() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_MESSAGE_ID.key(), "0:0");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectException(
+ topicName,
+ new IllegalArgumentException(
+ "MessageId format must be ledgerId:entryId:partitionId."));
+ }
+
+ @Test
+ void timestampStartCursor() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "233010230");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void messageIdStopCursorNever() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_STOP_AT_MESSAGE_ID.key(), "never");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void messageIdStopCursorLatest() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_STOP_AT_MESSAGE_ID.key(), "latest");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void messageIdStopCursorExact() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_STOP_AT_MESSAGE_ID.key(), "0:0:-1");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void timestampStopCursor() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SOURCE_STOP_AT_PUBLISH_TIME.key(), "233010230");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void topicRoutingMode() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SINK_TOPIC_ROUTING_MODE.key(), "message-key-hash");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void invalidTopicRouter() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ String invalidClassName = "invalid class name";
+ testConfigs.put(SINK_CUSTOM_TOPIC_ROUTER.key(), invalidClassName);
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName,
+ new ValidationException(
+ String.format(
+ "Could not find and instantiate TopicRouter class '%s'",
+ invalidClassName)));
+ }
+
+ @Test
+ void messageDelay() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "10s");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ @Test
+ void invalidMessageDelay() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(SINK_MESSAGE_DELAY_INTERVAL.key(), "invalid-duration");
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(
+ topicName,
+ new ValidationException("Invalid value for option 'sink.message-delay-interval'."));
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // PulsarOptions, PulsarSourceOptions, PulsarSinkOptions Test
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ void pulsarOptionsAuthParamMap() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put(PULSAR_AUTH_PARAM_MAP.key(), "key1:value1,key2:value2");
+ testConfigs.put(
+ PULSAR_AUTH_PLUGIN_CLASS_NAME.key(), MockPulsarAuthentication.class.getName());
+
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectSucceed(topicName);
+ runSourceAndExpectSucceed(topicName);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // requiredOptions(), optionalOptions() Test
+ // --------------------------------------------------------------------------------------------
+
+ @Test
+ void unusedConfigOptions() throws Exception {
+ final String topicName = randomTopicName();
+ Map<String, String> testConfigs = testConfigWithTopicAndFormat(topicName);
+
+ testConfigs.put("random_config", "random_value");
+ runSql(topicName, createTestConfig(testConfigs));
+ runSinkAndExpectException(topicName, ValidationException.class);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Utils methods
+ // --------------------------------------------------------------------------------------------
+
+ private String createTestConfig(Map<String, String> configMap) {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, String> entry : configMap.entrySet()) {
+ sb.append(String.format(" '%s' = '%s' ,\n", entry.getKey(), entry.getValue()));
+ }
+ return sb.toString();
+ }
+
+ private void runSql(String topicName, String testConfigString) throws Exception {
+ createTestTopic(topicName, 2);
+ final String createTable =
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `physical_1` STRING,\n"
+ + " `physical_2` INT,\n"
+ + " `physical_3` BOOLEAN\n"
+ + ") WITH (\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " %s\n"
+ + " 'connector' = 'pulsar'"
+ + ")",
+ topicName,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ testConfigString);
+ tableEnv.executeSql(createTable);
+ }
+
+ private void runSinkAndExpectSucceed(String topicName) {
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "VALUES\n"
+ + " ('data 1', 1, TRUE),\n"
+ + " ('data 2', 2, FALSE),\n"
+ + " ('data 3', 3, TRUE)",
+ topicName);
+ assertThatNoException().isThrownBy(() -> tableEnv.executeSql(initialValues).await());
+ }
+
+ private void runSinkAndExpectException(
+ String topicName, final Class<? extends Throwable> exceptionType) {
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "VALUES\n"
+ + " ('data 1', 1, TRUE),\n"
+ + " ('data 2', 2, FALSE),\n"
+ + " ('data 3', 3, TRUE)",
+ topicName);
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> tableEnv.executeSql(initialValues).await());
+ }
+
+ private void runSinkAndExpectException(String topicName, Throwable cause) {
+ String initialValues =
+ String.format(
+ "INSERT INTO %s\n"
+ + "VALUES\n"
+ + " ('data 1', 1, TRUE),\n"
+ + " ('data 2', 2, FALSE),\n"
+ + " ('data 3', 3, TRUE)",
+ topicName);
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> tableEnv.executeSql(initialValues).await())
+ .withCause(cause);
+ }
+
+ private void runSourceAndExpectSucceed(String topicName) {
+ assertThatNoException()
+ .isThrownBy(() -> tableEnv.sqlQuery(String.format("SELECT * FROM %s", topicName)));
+ }
+
+ private void runSourceAndExpectException(String topicName, Throwable cause) {
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> tableEnv.sqlQuery(String.format("SELECT * FROM %s", topicName)))
+ .withCause(cause);
+ }
+
+ private String randomTopicName() {
+ final String testTopicPrefix = "test_config_topic";
+ return testTopicPrefix + randomAlphabetic(5);
+ }
+
+ private Map<String, String> testConfigWithTopicAndFormat(String tableName) {
+ Map<String, String> testConfigs = new HashMap<>();
+ testConfigs.put(TOPICS.key(), tableName);
+ testConfigs.put(FactoryUtil.FORMAT.key(), "json");
+ return testConfigs;
+ }
+
+ private Map<String, String> testConfigWithFormat() {
+ Map<String, String> testConfigs = new HashMap<>();
+ testConfigs.put(FactoryUtil.FORMAT.key(), "json");
+ return testConfigs;
+ }
+
+ private Map<String, String> testConfigWithTopic(String tableName) {
+ Map<String, String> testConfigs = new HashMap<>();
+ testConfigs.put(TOPICS.key(), tableName);
+ return testConfigs;
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableTestBase.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableTestBase.java
new file mode 100644
index 0000000..58e2a35
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableTestBase.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.connector.pulsar.common.MiniClusterTestEnvironment;
+import org.apache.flink.connector.pulsar.testutils.PulsarTestEnvironment;
+import org.apache.flink.connector.pulsar.testutils.runtime.PulsarRuntime;
+import org.apache.flink.connector.testframe.junit.annotations.TestEnv;
+import org.apache.flink.connector.testframe.junit.annotations.TestExternalSystem;
+import org.apache.flink.connector.testframe.junit.annotations.TestSemantics;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
+
+/** Base class for Pulsar table integration test. */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class PulsarTableTestBase {
+ @TestEnv protected MiniClusterTestEnvironment flink = new MiniClusterTestEnvironment();
+
+ // Defines pulsar running environment
+ @TestExternalSystem
+ protected PulsarTestEnvironment pulsar = new PulsarTestEnvironment(PulsarRuntime.container());
+
+ @TestSemantics
+ protected CheckpointingMode[] semantics =
+ new CheckpointingMode[] {CheckpointingMode.EXACTLY_ONCE};
+
+ protected StreamExecutionEnvironment env;
+
+ protected StreamTableEnvironment tableEnv;
+
+ private static final int DEFAULT_PARALLELISM = 1;
+
+ @BeforeAll
+ public void beforeAll() throws Exception {
+ pulsar.startUp();
+ // run env
+ env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(DEFAULT_PARALLELISM);
+ env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
+ tableEnv = StreamTableEnvironment.create(env);
+ tableEnv.getConfig()
+ .getConfiguration()
+ .setString("table.dynamic-table-options.enabled", "true");
+ }
+
+ public void createTestTopic(String topic, int numPartitions) throws Exception {
+ pulsar.operator().createTopic(topic, numPartitions);
+ }
+
+ @AfterAll
+ public void afterAll() throws Exception {
+ pulsar.tearDown();
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtilsTest.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtilsTest.java
new file mode 100644
index 0000000..e3bba1b
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/PulsarTableValidationUtilsTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.table.api.ValidationException;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FIELDS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.KEY_FORMAT;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_CUSTOM_TOPIC_ROUTER;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SINK_TOPIC_ROUTING_MODE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_MESSAGE_ID;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_START_FROM_PUBLISH_TIME;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.SOURCE_SUBSCRIPTION_TYPE;
+import static org.apache.flink.connector.pulsar.table.PulsarTableOptions.TOPICS;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateKeyFormatConfigs;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateSinkRoutingConfigs;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateStartCursorConfigs;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateSubscriptionTypeConfigs;
+import static org.apache.flink.connector.pulsar.table.PulsarTableValidationUtils.validateTopicsConfigs;
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatNoException;
+
+/** Unit test for {@link PulsarTableValidationUtils}. */
+public class PulsarTableValidationUtilsTest {
+ @Test
+ void topicsConfigs() {
+ final Map<String, String> options = new HashMap<>();
+ options.put(TOPICS.key(), "");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateTopicsConfigs(Configuration.fromMap(options)))
+ .withMessage("The topics list should not be empty.");
+
+ String invalidTopicName = "persistent://tenant/topic";
+ String validTopicName = "valid-topic";
+
+ options.put(TOPICS.key(), invalidTopicName);
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateTopicsConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "The topics name %s is not a valid topic name.", invalidTopicName));
+
+ options.put(TOPICS.key(), validTopicName + ";" + invalidTopicName);
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateTopicsConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "The topics name %s is not a valid topic name.", invalidTopicName));
+
+ options.put(TOPICS.key(), validTopicName + ";" + validTopicName);
+ assertThatNoException()
+ .isThrownBy(() -> validateTopicsConfigs(Configuration.fromMap(options)));
+
+ options.put(TOPICS.key(), validTopicName + ";");
+ assertThatNoException()
+ .isThrownBy(() -> validateTopicsConfigs(Configuration.fromMap(options)));
+ }
+
+ @Test
+ void startCursorConfigs() {
+ final Map<String, String> options = new HashMap<>();
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "latest");
+ assertThatNoException()
+ .isThrownBy(() -> validateStartCursorConfigs(Configuration.fromMap(options)));
+
+ options.remove(SOURCE_START_FROM_MESSAGE_ID.key());
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "2345123234");
+ assertThatNoException()
+ .isThrownBy(() -> validateStartCursorConfigs(Configuration.fromMap(options)));
+
+ options.put(SOURCE_START_FROM_MESSAGE_ID.key(), "latest");
+ options.put(SOURCE_START_FROM_PUBLISH_TIME.key(), "2345123234");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateStartCursorConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "Only one of %s and %s can be specified. Detected both of them",
+ SOURCE_START_FROM_MESSAGE_ID, SOURCE_START_FROM_PUBLISH_TIME));
+ }
+
+ @Test
+ void subscriptionTypeConfigs() {
+ final Map<String, String> options = new HashMap<>();
+
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Exclusive");
+ assertThatNoException()
+ .isThrownBy(() -> validateSubscriptionTypeConfigs(Configuration.fromMap(options)));
+
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Shared");
+ assertThatNoException()
+ .isThrownBy(() -> validateSubscriptionTypeConfigs(Configuration.fromMap(options)));
+
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "Key_Shared");
+ assertThatNoException()
+ .isThrownBy(() -> validateSubscriptionTypeConfigs(Configuration.fromMap(options)));
+
+ options.put(SOURCE_SUBSCRIPTION_TYPE.key(), "invalid-subscription");
+ assertThatExceptionOfType(IllegalArgumentException.class)
+ .isThrownBy(() -> validateSubscriptionTypeConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ "Could not parse value 'invalid-subscription' for key 'source.subscription-type'.");
+ }
+
+ @Test
+ void sinkRoutingConfigs() {
+ final Map<String, String> options = new HashMap<>();
+ options.put(SINK_TOPIC_ROUTING_MODE.key(), "round-robin");
+ assertThatNoException()
+ .isThrownBy(() -> validateSinkRoutingConfigs(Configuration.fromMap(options)));
+
+ // validation does not try to create the class
+ options.remove(SINK_TOPIC_ROUTING_MODE.key());
+ options.put(SINK_CUSTOM_TOPIC_ROUTER.key(), "invalid-class-name");
+ assertThatNoException()
+ .isThrownBy(() -> validateSinkRoutingConfigs(Configuration.fromMap(options)));
+
+ options.put(SINK_TOPIC_ROUTING_MODE.key(), "round-robin");
+ options.put(SINK_CUSTOM_TOPIC_ROUTER.key(), "invalid-class-name");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateSinkRoutingConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "Only one of %s and %s can be specified. Detected both of them",
+ SINK_CUSTOM_TOPIC_ROUTER, SINK_TOPIC_ROUTING_MODE));
+ }
+
+ @Test
+ void keyFormatConfigs() {
+ final Map<String, String> options = new HashMap<>();
+ options.put(KEY_FIELDS.key(), "");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateKeyFormatConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "The option '%s' can only be declared if a key format is defined using '%s'.",
+ KEY_FIELDS.key(), KEY_FORMAT.key()));
+
+ options.put(KEY_FORMAT.key(), "json");
+ options.remove(KEY_FIELDS.key());
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateKeyFormatConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "A key format '%s' requires the declaration of one or more of key fields using '%s'.",
+ KEY_FORMAT.key(), KEY_FIELDS.key()));
+
+ options.put(KEY_FORMAT.key(), "json");
+ options.put(KEY_FIELDS.key(), "");
+ assertThatExceptionOfType(ValidationException.class)
+ .isThrownBy(() -> validateKeyFormatConfigs(Configuration.fromMap(options)))
+ .withMessage(
+ String.format(
+ "A key format '%s' requires the declaration of one or more of key fields using '%s'.",
+ KEY_FORMAT.key(), KEY_FIELDS.key()));
+
+ options.put(KEY_FORMAT.key(), "json");
+ options.put(KEY_FIELDS.key(), "k_field1");
+ assertThatNoException()
+ .isThrownBy(() -> validateKeyFormatConfigs(Configuration.fromMap(options)));
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableITCase.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableITCase.java
new file mode 100644
index 0000000..0173192
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/UpsertPulsarTableITCase.java
@@ -0,0 +1,573 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table;
+
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.planner.factories.TestValuesTableFactory;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+import org.apache.flink.types.Row;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.collectRows;
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.comparedWithKeyAndOrder;
+import static org.apache.flink.connector.pulsar.table.testutils.PulsarTableTestUtils.waitingExpectedResults;
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * IT cases for the Pulsar table source and sink. It aims to verify runtime behaviour and certain
+ * use cases are correct and can produce/consume the desired records as user specifies.
+ */
+@ExtendWith(MiniClusterExtension.class)
+public class UpsertPulsarTableITCase extends PulsarTableITCase {
+ private static final String USERS_TOPIC = "users";
+ private static final String WORD_COUNT_TOPIC = "word_count";
+
+ @ParameterizedTest
+ @ValueSource(strings = {JSON_FORMAT})
+ public void testAggregate(String format) throws Exception {
+ String topic = WORD_COUNT_TOPIC + "_" + format;
+ pulsar.operator().createTopic(topic, 4);
+ // ------------- test ---------------
+ wordCountToUpsertPulsar(format, topic);
+ wordFreqToUpsertPulsar(format, topic);
+ }
+
+ /**
+ * There is a known issue when parallelism is set to 2 in the temporal join. This is related to
+ * the watermark of Pulsar source. The partition number is expected to be the same as the
+ * parallelism in this test.
+ */
+ @ParameterizedTest
+ @ValueSource(strings = {JSON_FORMAT})
+ public void testTemporalJoin(String format) throws Exception {
+ String topic = USERS_TOPIC + "_" + format;
+ pulsar.operator().createTopic(topic, 1);
+
+ env.setParallelism(1);
+ writeChangelogToUpsertPulsarWithMetadata(format, topic);
+ env.setParallelism(1);
+ temporalJoinUpsertPulsar(format, topic);
+ }
+
+ private void wordCountToUpsertPulsar(String format, String wordCountTable) throws Exception {
+ // ------------- test data ---------------
+
+ final List<Row> inputData =
+ Arrays.stream("Good good study day day up Good boy".split(" "))
+ .map(Row::of)
+ .collect(Collectors.toList());
+
+ // ------------- create table ---------------
+
+ final String createSource =
+ String.format(
+ "CREATE TABLE words_%s ("
+ + " `word` STRING"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'data-id' = '%s'"
+ + ")",
+ format, TestValuesTableFactory.registerData(inputData));
+ tableEnv.executeSql(createSource);
+ final String createSinkTable =
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `word` STRING,\n"
+ + " `count` BIGINT,\n"
+ + " PRIMARY KEY (`word`) NOT ENFORCED\n"
+ + ") WITH (\n"
+ + " 'connector' = 'upsert-pulsar',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'key.format' = '%s',\n"
+ + " 'value.format' = '%s'"
+ + ")",
+ wordCountTable,
+ wordCountTable,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format,
+ format);
+ tableEnv.executeSql(createSinkTable);
+ String initialValues =
+ "INSERT INTO "
+ + wordCountTable
+ + " "
+ + "SELECT LOWER(`word`), count(*) as `count` "
+ + "FROM words_"
+ + format
+ + " "
+ + "GROUP BY LOWER(`word`)";
+ tableEnv.executeSql(initialValues).await();
+
+ // ---------- read from the upsert sink -------------------
+
+ final List<Row> result =
+ collectRows(tableEnv.sqlQuery("SELECT * FROM " + wordCountTable), 11);
+
+ final Map<Row, List<Row>> expected = new HashMap<>();
+ expected.put(
+ Row.of("good"),
+ Arrays.asList(
+ changelogRow("+I", "good", 1L),
+ changelogRow("-U", "good", 1L),
+ changelogRow("+U", "good", 2L),
+ changelogRow("-U", "good", 2L),
+ changelogRow("+U", "good", 3L)));
+ expected.put(Row.of("study"), Collections.singletonList(changelogRow("+I", "study", 1L)));
+ expected.put(
+ Row.of("day"),
+ Arrays.asList(
+ changelogRow("+I", "day", 1L),
+ changelogRow("-U", "day", 1L),
+ changelogRow("+U", "day", 2L)));
+ expected.put(Row.of("up"), Collections.singletonList(changelogRow("+I", "up", 1L)));
+ expected.put(Row.of("boy"), Collections.singletonList(changelogRow("+I", "boy", 1L)));
+
+ comparedWithKeyAndOrder(expected, result, new int[] {0});
+
+ // ---------- read the raw data from pulsar -------------------
+ // check we only write the upsert data into Pulsar
+ String rawWordCountTable = "raw_word_count";
+ tableEnv.executeSql(
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `word` STRING NOT NULL,\n"
+ + " `count` BIGINT\n"
+ + ") WITH (\n"
+ + " 'connector' = 'pulsar',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'key.format' = '%s',\n"
+ + " 'key.fields' = 'word',\n"
+ + " 'value.format' = '%s'\n"
+ + ")",
+ rawWordCountTable,
+ wordCountTable,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format,
+ format));
+
+ final List<Row> result2 =
+ collectRows(tableEnv.sqlQuery("SELECT * FROM " + rawWordCountTable), 8);
+ final Map<Row, List<Row>> expected2 = new HashMap<>();
+ expected2.put(
+ Row.of("good"),
+ Arrays.asList(Row.of("good", 1L), Row.of("good", 2L), Row.of("good", 3L)));
+ expected2.put(Row.of("study"), Collections.singletonList(Row.of("study", 1L)));
+ expected2.put(Row.of("day"), Arrays.asList(Row.of("day", 1L), Row.of("day", 2L)));
+ expected2.put(Row.of("up"), Collections.singletonList(Row.of("up", 1L)));
+ expected2.put(Row.of("boy"), Collections.singletonList(Row.of("boy", 1L)));
+
+ comparedWithKeyAndOrder(expected2, result2, new int[] {0});
+ }
+
+ private void wordFreqToUpsertPulsar(String format, String wordCountTable) throws Exception {
+ // ------------- test data ---------------
+
+ final List<String> expectedData = Arrays.asList("+I[3, 1]", "+I[2, 1]");
+
+ // ------------- create table ---------------
+
+ final String createSinkTable =
+ "CREATE TABLE sink_"
+ + format
+ + "(\n"
+ + " `count` BIGINT,\n"
+ + " `freq` BIGINT,\n"
+ + " PRIMARY KEY (`count`) NOT ENFORCED\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values',\n"
+ + " 'sink-insert-only' = 'false'\n"
+ + ")";
+ tableEnv.executeSql(createSinkTable);
+ final TableResult query =
+ tableEnv.executeSql(
+ "INSERT INTO sink_"
+ + format
+ + "\n"
+ + "SELECT `count`, count(*) as `freq`\n"
+ + "FROM "
+ + wordCountTable
+ + "\n"
+ + "GROUP BY `count`\n"
+ + "having count(*) < 2");
+
+ waitingExpectedResults("sink_" + format, expectedData, Duration.ofSeconds(20));
+ query.getJobClient().get().cancel();
+ }
+
+ private void writeChangelogToUpsertPulsarWithMetadata(String format, String userTable)
+ throws Exception {
+ // ------------- test data ---------------
+
+ // Prepare data for upsert pulsar
+ // Keep every partition has more than 1 record and the last record in every partition has
+ // larger event time
+ // than left stream event time to trigger the join.
+ List<Row> changelogData =
+ Arrays.asList(
+ changelogRow(
+ "+U",
+ 100L,
+ "Bob",
+ "Beijing",
+ LocalDateTime.parse("2020-08-15T00:00:01")),
+ changelogRow(
+ "+U",
+ 101L,
+ "Alice",
+ "Shanghai",
+ LocalDateTime.parse("2020-08-15T00:00:02")),
+ changelogRow(
+ "+U",
+ 102L,
+ "Greg",
+ "Berlin",
+ LocalDateTime.parse("2020-08-15T00:00:03")),
+ changelogRow(
+ "+U",
+ 103L,
+ "Richard",
+ "Berlin",
+ LocalDateTime.parse("2020-08-16T00:01:05")),
+ changelogRow(
+ "+U",
+ 101L,
+ "Alice",
+ "Wuhan",
+ LocalDateTime.parse("2020-08-16T00:02:00")),
+ changelogRow(
+ "+U",
+ 104L,
+ "Tomato",
+ "Hongkong",
+ LocalDateTime.parse("2020-08-16T00:05:05")),
+ changelogRow(
+ "+U",
+ 105L,
+ "Tim",
+ "Shenzhen",
+ LocalDateTime.parse("2020-08-16T00:06:00")),
+ // Keep the timestamp in the records are in the ascending order.
+ // It will keep the records in the pulsar partition are in the order.
+ // It has the same effects by adjusting the watermark strategy.
+ changelogRow(
+ "+U",
+ 103L,
+ "Richard",
+ "London",
+ LocalDateTime.parse("2020-08-16T01:01:05")),
+ changelogRow(
+ "+U",
+ 101L,
+ "Alice",
+ "Hangzhou",
+ LocalDateTime.parse("2020-08-16T01:04:05")),
+ changelogRow(
+ "+U",
+ 104L,
+ "Tomato",
+ "Shenzhen",
+ LocalDateTime.parse("2020-08-16T01:05:05")),
+ changelogRow(
+ "+U",
+ 105L,
+ "Tim",
+ "Hongkong",
+ LocalDateTime.parse("2020-08-16T01:06:00")));
+
+ // ------------- create table ---------------
+
+ final String createChangelog =
+ String.format(
+ "CREATE TABLE users_changelog_%s ("
+ + " user_id BIGINT,"
+ + " user_name STRING,"
+ + " region STRING,"
+ + " modification_time TIMESTAMP(3),"
+ + " PRIMARY KEY (user_id) NOT ENFORCED"
+ + ") WITH ("
+ + " 'connector' = 'values',"
+ + " 'data-id' = '%s',"
+ + " 'changelog-mode' = 'UA,D',"
+ + " 'disable-lookup' = 'true'"
+ + ")",
+ format, TestValuesTableFactory.registerData(changelogData));
+ tableEnv.executeSql(createChangelog);
+
+ // we verified computed column, watermark, metadata in this case too
+ final String createSinkTable =
+ String.format(
+ "CREATE TABLE %s (\n"
+ + " `user_id` BIGINT,\n"
+ + " `user_name` STRING,\n"
+ + " `region` STRING,\n"
+ + " upper_region AS UPPER(`region`),\n"
+ + " modification_time TIMESTAMP(3) METADATA FROM 'event_time',\n"
+ + " watermark for modification_time as modification_time,\n"
+ + " PRIMARY KEY (`user_id`) NOT ENFORCED\n"
+ + ") WITH (\n"
+ + " 'connector' = 'upsert-pulsar',\n"
+ + " 'topics' = '%s',\n"
+ + " 'service-url' = '%s',\n"
+ + " 'admin-url' = '%s',\n"
+ + " 'key.format' = '%s',\n"
+ + " 'value.format' = '%s'"
+ + ")",
+ userTable,
+ userTable,
+ pulsar.operator().serviceUrl(),
+ pulsar.operator().adminUrl(),
+ format,
+ format);
+ tableEnv.executeSql(createSinkTable);
+ String initialValues =
+ "INSERT INTO " + userTable + " " + "SELECT * " + "FROM users_changelog_" + format;
+ tableEnv.executeSql(initialValues).await();
+
+ // ---------- consume stream from sink -------------------
+
+ final List<Row> result = collectRows(tableEnv.sqlQuery("SELECT * FROM " + userTable), 16);
+
+ List<Row> expected =
+ Arrays.asList(
+ changelogRow(
+ "+I",
+ 100L,
+ "Bob",
+ "Beijing",
+ "BEIJING",
+ LocalDateTime.parse("2020-08-15T00:00:01")),
+ changelogRow(
+ "+I",
+ 101L,
+ "Alice",
+ "Shanghai",
+ "SHANGHAI",
+ LocalDateTime.parse("2020-08-15T00:00:02")),
+ changelogRow(
+ "-U",
+ 101L,
+ "Alice",
+ "Shanghai",
+ "SHANGHAI",
+ LocalDateTime.parse("2020-08-15T00:00:02")),
+ changelogRow(
+ "+U",
+ 101L,
+ "Alice",
+ "Wuhan",
+ "WUHAN",
+ LocalDateTime.parse("2020-08-16T00:02:00")),
+ changelogRow(
+ "-U",
+ 101L,
+ "Alice",
+ "Wuhan",
+ "WUHAN",
+ LocalDateTime.parse("2020-08-16T00:02:00")),
+ changelogRow(
+ "+U",
+ 101L,
+ "Alice",
+ "Hangzhou",
+ "HANGZHOU",
+ LocalDateTime.parse("2020-08-16T01:04:05")),
+ changelogRow(
+ "+I",
+ 102L,
+ "Greg",
+ "Berlin",
+ "BERLIN",
+ LocalDateTime.parse("2020-08-15T00:00:03")),
+ changelogRow(
+ "+I",
+ 103L,
+ "Richard",
+ "Berlin",
+ "BERLIN",
+ LocalDateTime.parse("2020-08-16T00:01:05")),
+ changelogRow(
+ "-U",
+ 103L,
+ "Richard",
+ "Berlin",
+ "BERLIN",
+ LocalDateTime.parse("2020-08-16T00:01:05")),
+ changelogRow(
+ "+U",
+ 103L,
+ "Richard",
+ "London",
+ "LONDON",
+ LocalDateTime.parse("2020-08-16T01:01:05")),
+ changelogRow(
+ "+I",
+ 104L,
+ "Tomato",
+ "Hongkong",
+ "HONGKONG",
+ LocalDateTime.parse("2020-08-16T00:05:05")),
+ changelogRow(
+ "-U",
+ 104L,
+ "Tomato",
+ "Hongkong",
+ "HONGKONG",
+ LocalDateTime.parse("2020-08-16T00:05:05")),
+ changelogRow(
+ "+U",
+ 104L,
+ "Tomato",
+ "Shenzhen",
+ "SHENZHEN",
+ LocalDateTime.parse("2020-08-16T01:05:05")),
+ changelogRow(
+ "+I",
+ 105L,
+ "Tim",
+ "Shenzhen",
+ "SHENZHEN",
+ LocalDateTime.parse("2020-08-16T00:06")),
+ changelogRow(
+ "-U",
+ 105L,
+ "Tim",
+ "Shenzhen",
+ "SHENZHEN",
+ LocalDateTime.parse("2020-08-16T00:06")),
+ changelogRow(
+ "+U",
+ 105L,
+ "Tim",
+ "Hongkong",
+ "HONGKONG",
+ LocalDateTime.parse("2020-08-16T01:06")));
+ assertThat(result).hasSameElementsAs(expected);
+ }
+
+ private void temporalJoinUpsertPulsar(String format, String userTable) throws Exception {
+ // ------------- test data ---------------
+ List<Row> input =
+ Arrays.asList(
+ Row.of(10001L, 100L, LocalDateTime.parse("2020-08-15T00:00:02")),
+ Row.of(10002L, 101L, LocalDateTime.parse("2020-08-15T00:00:03")),
+ Row.of(10002L, 102L, LocalDateTime.parse("2020-08-15T00:00:04")),
+ Row.of(10002L, 101L, LocalDateTime.parse("2020-08-16T00:02:01")),
+ Row.of(10004L, 104L, LocalDateTime.parse("2020-08-16T00:04:00")),
+ Row.of(10003L, 101L, LocalDateTime.parse("2020-08-16T00:04:06")),
+ Row.of(10004L, 104L, LocalDateTime.parse("2020-08-16T00:05:06")));
+
+ List<Row> expected =
+ Arrays.asList(
+ Row.of(
+ 10001L,
+ 100L,
+ LocalDateTime.parse("2020-08-15T00:00:02"),
+ "Bob",
+ "BEIJING",
+ LocalDateTime.parse("2020-08-15T00:00:01")),
+ Row.of(
+ 10002L,
+ 101L,
+ LocalDateTime.parse("2020-08-15T00:00:03"),
+ "Alice",
+ "SHANGHAI",
+ LocalDateTime.parse("2020-08-15T00:00:02")),
+ Row.of(
+ 10002L,
+ 102L,
+ LocalDateTime.parse("2020-08-15T00:00:04"),
+ "Greg",
+ "BERLIN",
+ LocalDateTime.parse("2020-08-15T00:00:03")),
+ Row.of(
+ 10002L,
+ 101L,
+ LocalDateTime.parse("2020-08-16T00:02:01"),
+ "Alice",
+ "WUHAN",
+ LocalDateTime.parse("2020-08-16T00:02:00")),
+ Row.of(
+ 10004L,
+ 104L,
+ LocalDateTime.parse("2020-08-16T00:04:00"),
+ null,
+ null,
+ null),
+ Row.of(
+ 10003L,
+ 101L,
+ LocalDateTime.parse("2020-08-16T00:04:06"),
+ "Alice",
+ "WUHAN",
+ LocalDateTime.parse("2020-08-16T00:02:00")),
+ Row.of(
+ 10004L,
+ 104L,
+ LocalDateTime.parse("2020-08-16T00:05:06"),
+ "Tomato",
+ "HONGKONG",
+ LocalDateTime.parse("2020-08-16T00:05:05")));
+
+ // ------------- create table ---------------
+
+ tableEnv.executeSql(
+ String.format(
+ "CREATE TABLE pageviews_%s(\n"
+ + " `page_id` BIGINT,\n"
+ + " `user_id` BIGINT,\n"
+ + " `viewtime` TIMESTAMP(3),\n"
+ + " `proctime` as proctime(),\n"
+ + " watermark for `viewtime` as `viewtime`\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values',\n"
+ + " 'data-id' = '%s'\n"
+ + ")",
+ format, TestValuesTableFactory.registerData(input)));
+ final List<Row> result =
+ collectRows(
+ tableEnv.sqlQuery(
+ String.format(
+ "SELECT p.page_id, p.user_id, p.viewtime, u.user_name, u.upper_region, u.modification_time\n"
+ + "FROM pageviews_%s AS p\n"
+ + "LEFT JOIN %s FOR SYSTEM_TIME AS OF p.viewtime AS u\n"
+ + "ON p.user_id = u.user_id",
+ format, userTable)),
+ 7);
+ assertThat(result).hasSameElementsAs(expected);
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockPulsarAuthentication.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockPulsarAuthentication.java
new file mode 100644
index 0000000..cbb2a86
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockPulsarAuthentication.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.testutils;
+
+import org.apache.pulsar.client.api.Authentication;
+import org.apache.pulsar.client.api.AuthenticationDataProvider;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.impl.auth.AuthenticationDataNull;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+/** A class to verify Pulsar authentication auth params map is created properly. way to do this */
+public class MockPulsarAuthentication implements Authentication {
+ public static String key1 = "key1";
+ public static String key2 = "key2";
+ public static String value1 = "value1";
+ public static String value2 = "value2";
+
+ @Override
+ public String getAuthMethodName() {
+ return "custom authentication";
+ }
+
+ @Override
+ public AuthenticationDataProvider getAuthData() {
+ return new AuthenticationDataNull();
+ }
+
+ @Override
+ public AuthenticationDataProvider getAuthData(String brokerHostName) {
+ return new AuthenticationDataNull();
+ }
+
+ @Override
+ public void authenticationStage(
+ String requestUrl,
+ AuthenticationDataProvider authData,
+ Map<String, String> previousResHeaders,
+ CompletableFuture<Map<String, String>> authFuture) {
+ Authentication.super.authenticationStage(
+ requestUrl, authData, previousResHeaders, authFuture);
+ }
+
+ @Override
+ public Set<Map.Entry<String, String>> newRequestHeader(
+ String hostName,
+ AuthenticationDataProvider authData,
+ Map<String, String> previousResHeaders) {
+ return new HashSet<>();
+ }
+
+ @Override
+ public void configure(Map<String, String> authParams) {
+ assert Objects.equals(authParams.get(key1), value1);
+ assert Objects.equals(authParams.get(key2), value2);
+ }
+
+ @Override
+ public void start() throws PulsarClientException {}
+
+ @Override
+ public void close() throws IOException {}
+}
diff --git a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockTopicRouter.java
similarity index 51%
copy from flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java
copy to flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockTopicRouter.java
index da51a58..183ce92 100644
--- a/flink-connector-pulsar/src/main/java/org/apache/flink/connector/pulsar/source/enumerator/cursor/start/TimestampStartCursor.java
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/MockTopicRouter.java
@@ -16,23 +16,26 @@
* limitations under the License.
*/
-package org.apache.flink.connector.pulsar.source.enumerator.cursor.start;
+package org.apache.flink.connector.pulsar.table.testutils;
-import org.apache.flink.connector.pulsar.source.enumerator.cursor.CursorPosition;
-import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
+import org.apache.flink.connector.pulsar.sink.writer.context.PulsarSinkContext;
+import org.apache.flink.connector.pulsar.sink.writer.router.TopicRouter;
+import org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition;
+import org.apache.flink.table.data.RowData;
-/** This cursor would left pulsar start consuming from a specific publish timestamp. */
-public class TimestampStartCursor implements StartCursor {
- private static final long serialVersionUID = 5170578885838095320L;
+import java.util.List;
- private final long timestamp;
+/** A mock topic Router for testing purposes only. */
+public class MockTopicRouter implements TopicRouter<RowData> {
- public TimestampStartCursor(long timestamp, boolean inclusive) {
- this.timestamp = inclusive ? timestamp : timestamp + 1;
- }
+ private static final long serialVersionUID = 1316133122715449818L;
@Override
- public CursorPosition position(String topic, int partitionId) {
- return new CursorPosition(timestamp);
+ public TopicPartition route(
+ RowData rowData,
+ String key,
+ List<TopicPartition> partitions,
+ PulsarSinkContext context) {
+ return new TopicPartition("never-exist-topic");
}
}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/PulsarTableTestUtils.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/PulsarTableTestUtils.java
new file mode 100644
index 0000000..fe29b4b
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/PulsarTableTestUtils.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.pulsar.table.testutils;
+
+import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.planner.factories.TestValuesTableFactory;
+import org.apache.flink.types.Row;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.types.RowUtils;
+import org.apache.flink.util.CloseableIterator;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Util class for verify testing results. */
+public class PulsarTableTestUtils {
+ public static List<Row> collectRows(Table table, int expectedSize) throws Exception {
+ final TableResult result = table.execute();
+ final List<Row> collectedRows = new ArrayList<>();
+ try (CloseableIterator<Row> iterator = result.collect()) {
+ while (collectedRows.size() < expectedSize && iterator.hasNext()) {
+ collectedRows.add(iterator.next());
+ }
+ }
+ result.getJobClient()
+ .ifPresent(
+ jc -> {
+ try {
+ jc.cancel().get(5, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ return collectedRows;
+ }
+
+ public static void comparedWithKeyAndOrder(
+ Map<Row, List<Row>> expectedData, List<Row> actual, int[] keyLoc) {
+ Map<Row, LinkedList<Row>> actualData = new HashMap<>();
+ for (Row row : actual) {
+ Row key = Row.project(row, keyLoc);
+ // ignore row kind
+ key.setKind(RowKind.INSERT);
+ actualData.computeIfAbsent(key, k -> new LinkedList<>()).add(row);
+ }
+ // compare key first
+ assertThat(actualData).hasSameSizeAs(expectedData);
+ // compare by value
+ for (Row key : expectedData.keySet()) {
+ assertThat(actualData.get(key))
+ .withFailMessage(
+ "\nexpected: %s\n but was: %s\n for key: %s",
+ expectedData.get(key), actualData.get(key), key)
+ .matches(
+ rows ->
+ RowUtils.compareRows(
+ expectedData.get(key), (List<Row>) rows, false));
+ }
+ }
+
+ public static void waitingExpectedResults(
+ String sinkName, List<String> expected, Duration timeout)
+ throws InterruptedException, TimeoutException {
+ Collections.sort(expected);
+ CommonTestUtils.waitUtil(
+ () -> {
+ List<String> actual = TestValuesTableFactory.getResults(sinkName);
+ Collections.sort(actual);
+ return expected.equals(actual);
+ },
+ timeout,
+ "Can not get the expected result.");
+ }
+
+ public static List<String> readLines(String resource) throws IOException {
+ final URL url = PulsarTableTestUtils.class.getClassLoader().getResource(resource);
+ assert url != null;
+ Path path = new File(url.getFile()).toPath();
+ return Files.readAllLines(path);
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/TestingUser.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/TestingUser.java
new file mode 100644
index 0000000..b597f3f
--- /dev/null
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/table/testutils/TestingUser.java
@@ -0,0 +1,61 @@
+package org.apache.flink.connector.pulsar.table.testutils;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
+
+/** A test POJO class. */
+public class TestingUser implements Serializable {
+ private static final long serialVersionUID = -1123545861004770003L;
+ public String name;
+ public Integer age;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getAge() {
+ return age;
+ }
+
+ public void setAge(Integer age) {
+ this.age = age;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TestingUser that = (TestingUser) o;
+ return Objects.equals(name, that.name) && Objects.equals(age, that.age);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, age);
+ }
+
+ public static TestingUser createUser(String name, Integer age) {
+ TestingUser user = new TestingUser();
+ user.setAge(age);
+ user.setName(name);
+ return user;
+ }
+
+ public static TestingUser createRandomUser() {
+ TestingUser user = new TestingUser();
+ user.setName(randomAlphabetic(5));
+ user.setAge(ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE));
+ return user;
+ }
+}
diff --git a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/PulsarRuntimeOperator.java b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/PulsarRuntimeOperator.java
index 71d0f98..bd4b243 100644
--- a/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/PulsarRuntimeOperator.java
+++ b/flink-connector-pulsar/src/test/java/org/apache/flink/connector/pulsar/testutils/runtime/PulsarRuntimeOperator.java
@@ -481,7 +481,7 @@ public class PulsarRuntimeOperator implements Closeable {
}
}
- private <T> Producer<T> createProducer(String topic, Schema<T> schema) throws Exception {
+ public <T> Producer<T> createProducer(String topic, Schema<T> schema) throws Exception {
return client().newProducer(schema)
.topic(topic)
.enableBatching(false)
@@ -490,7 +490,7 @@ public class PulsarRuntimeOperator implements Closeable {
.create();
}
- private <T> Consumer<T> createConsumer(String topic, Schema<T> schema) throws Exception {
+ public <T> Consumer<T> createConsumer(String topic, Schema<T> schema) throws Exception {
// Create the earliest subscription if it's not existed.
List<String> subscriptions = admin().topics().getSubscriptions(topic);
if (!subscriptions.contains(SUBSCRIPTION_NAME)) {
diff --git a/flink-connector-pulsar/src/test/resources/canal-data.txt b/flink-connector-pulsar/src/test/resources/canal-data.txt
new file mode 100644
index 0000000..0f74e28
--- /dev/null
+++ b/flink-connector-pulsar/src/test/resources/canal-data.txt
@@ -0,0 +1,11 @@
+{"data":[{"id":"101","name":"scooter","description":"Small 2-wheel scooter","weight":"3.14"},{"id":"102","name":"car battery","description":"12V car battery","weight":"8.1"},{"id":"103","name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":"0.8"},{"id":"104","name":"hammer","description":"12oz carpenter's hammer","weight":"0.75"},{"id":"105","name":"hammer","description":"14oz carpenter's hammer","weight":"0.875"},{"id":"106","name": [...]
+{"data":[{"id":"106","name":"hammer","description":"18oz carpenter hammer","weight":"1.0"}],"database":"inventory","es":1589373546000,"id":4,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":[{"description":"16oz carpenter's hammer"}],"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373546301,"type":"UPDATE"}
+{"data":[{"id":"107","name":"rocks","description":"box of assorted rocks","weight":"5.1"}],"database":"inventory","es":1589373549000,"id":5,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":[{"weight":"5.3"}],"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373549489,"type":"UPDATE"}
+{"data":[{"id":"110","name":"jacket","description":"water resistent white wind breaker","weight":"0.2"}],"database":"inventory","es":1589373552000,"id":6,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":null,"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373552882,"type":"INSERT"}
+{"data":[{"id":"111","name":"scooter","description":"Big 2-wheel scooter ","weight":"5.18"}],"database":"inventory","es":1589373555000,"id":7,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":null,"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373555457,"type":"INSERT"}
+{"data":[{"id":"110","name":"jacket","description":"new water resistent white wind breaker","weight":"0.5"}],"database":"inventory","es":1589373558000,"id":8,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":[{"description":"water resistent white wind breaker","weight":"0.2"}],"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373558230,"type":"UPDATE"}
+{"data":[{"id":"111","name":"scooter","description":"Big 2-wheel scooter ","weight":"5.17"}],"database":"inventory","es":1589373560000,"id":9,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":[{"weight":"5.18"}],"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373560798,"type":"UPDATE"}
+{"data":[{"id":"111","name":"scooter","description":"Big 2-wheel scooter ","weight":"5.17"}],"database":"inventory","es":1589373563000,"id":10,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":null,"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589373563798,"type":"DELETE"}
+{"data":[{"id":"101","name":"scooter","description":"Small 2-wheel scooter","weight":"5.17"},{"id":"102","name":"car battery","description":"12V car battery","weight":"5.17"}],"database":"inventory","es":1589373753000,"id":11,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":[{"weight":"3.14"},{"weight":"8.1"}],"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products2","ts":1589 [...]
+{"data":null,"database":"inventory","es":1589373566000,"id":13,"isDdl":true,"mysqlType":null,"old":null,"pkNames":null,"sql":"CREATE TABLE `xj_`.`user02` (`uid` int(0) NOT NULL,`uname` varchar(255) NULL, PRIMARY KEY (`uid`))","sqlType":null,"table":"user02","ts":1589373566000,"type":"CREATE"}
+{"data":[{"id":"102","name":"car battery","description":"12V car battery","weight":"5.17"},{"id":"103","name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":"0.8"}],"database":"inventory","es":1589374013000,"id":12,"isDdl":false,"mysqlType":{"id":"INTEGER","name":"VARCHAR(255)","description":"VARCHAR(512)","weight":"FLOAT"},"old":null,"pkNames":["id"],"sql":"","sqlType":{"id":4,"name":12,"description":12,"weight":7},"table":"products [...]
\ No newline at end of file
diff --git a/flink-connector-pulsar/src/test/resources/debezium-data-schema-exclude.txt b/flink-connector-pulsar/src/test/resources/debezium-data-schema-exclude.txt
new file mode 100644
index 0000000..3763369
--- /dev/null
+++ b/flink-connector-pulsar/src/test/resources/debezium-data-schema-exclude.txt
@@ -0,0 +1,16 @@
+{"before":null,"after":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.140000104904175},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606100,"transaction":null}
+{"before":null,"after":{"id":102,"name":"car battery","description":"12V car battery","weight":8.100000381469727},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.800000011920929},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":106,"name":"hammer","description":"16oz carpenter's hammer","weight":1},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.300000190734863},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":108,"name":"jacket","description":"water resistent black wind breaker","weight":0.10000000149011612},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":null,"after":{"id":109,"name":"spare tire","description":"24 inch spare tire","weight":22.200000762939453},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":0,"snapshot":"true","db":"inventory","table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":154,"row":0,"thread":null,"query":null},"op":"c","ts_ms":1589355606101,"transaction":null}
+{"before":{"id":106,"name":"hammer","description":"16oz carpenter's hammer","weight":1},"after":{"id":106,"name":"hammer","description":"18oz carpenter hammer","weight":1},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589361987000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":362,"row":0,"thread":2,"query":null},"op":"u","ts_ms":1589361987936,"transaction":null}
+{"before":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.300000190734863},"after":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.099999904632568},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362099000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":717,"row":0,"thread":2,"query":null},"op":"u","ts_ms":1589362099505,"transactio [...]
+{"before":null,"after":{"id":110,"name":"jacket","description":"water resistent white wind breaker","weight":0.20000000298023224},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362210000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":1068,"row":0,"thread":2,"query":null},"op":"c","ts_ms":1589362210230,"transaction":null}
+{"before":null,"after":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.179999828338623},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362243000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":1394,"row":0,"thread":2,"query":null},"op":"c","ts_ms":1589362243428,"transaction":null}
+{"before":{"id":110,"name":"jacket","description":"water resistent white wind breaker","weight":0.20000000298023224},"after":{"id":110,"name":"jacket","description":"new water resistent white wind breaker","weight":0.5},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362293000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":1707,"row":0,"thread":2,"query":null},"op":"u","ts_ms":1589 [...]
+{"before":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.179999828338623},"after":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.170000076293945},"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362330000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":2090,"row":0,"thread":2,"query":null},"op":"u","ts_ms":1589362330904,"transac [...]
+{"before":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.170000076293945},"after":null,"source":{"version":"1.1.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1589362344000,"snapshot":"false","db":"inventory","table":"products","server_id":223344,"gtid":null,"file":"mysql-bin.000003","pos":2443,"row":0,"thread":2,"query":null},"op":"d","ts_ms":1589362344455,"transaction":null}
\ No newline at end of file
diff --git a/flink-connector-pulsar/src/test/resources/log4j2-test.properties b/flink-connector-pulsar/src/test/resources/log4j2-test.properties
index b6a9837..6e75b82 100644
--- a/flink-connector-pulsar/src/test/resources/log4j2-test.properties
+++ b/flink-connector-pulsar/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level=OFF
+rootLogger.level=INFO
rootLogger.appenderRef.test.ref=TestLogger
appender.testlogger.name=TestLogger
appender.testlogger.type=CONSOLE
@@ -28,8 +28,8 @@ appender.testlogger.layout.pattern=%-4r [%t] %-5p %c %x - %m%n
# Logger for pulsar.
logger.pulsar.name=org.apache.pulsar
-logger.pulsar.level=OFF
+logger.pulsar.level=INFO
# Logger for pulsar connector.
logger.pulsar.connector.name=org.apache.flink.connector.pulsar
-logger.pulsar.connector.level=OFF
+logger.pulsar.connector.level=INFO
diff --git a/flink-connector-pulsar/src/test/resources/maxwell-data.txt b/flink-connector-pulsar/src/test/resources/maxwell-data.txt
new file mode 100644
index 0000000..ecba573
--- /dev/null
+++ b/flink-connector-pulsar/src/test/resources/maxwell-data.txt
@@ -0,0 +1,20 @@
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":1,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":2,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":3,"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":4,"data":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":5,"data":{"id":106,"name":"hammer","description":"16oz carpenter's hammer","weight":1.0}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":6,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":7,"data":{"id":108,"name":"jacket","description":"water resistent black wind breaker","weight":0.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"commit":true,"data":{"id":109,"name":"spare tire","description":"24 inch spare tire","weight":22.2}}
+{"database":"test","table":"product","type":"update","ts":1596684893,"xid":7152,"commit":true,"data":{"id":106,"name":"hammer","description":"18oz carpenter hammer","weight":1.0},"old":{"description":"16oz carpenter's hammer"}}
+{"database":"test","table":"product","type":"update","ts":1596684897,"xid":7169,"commit":true,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.1},"old":{"weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684900,"xid":7186,"commit":true,"data":{"id":110,"name":"jacket","description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"insert","ts":1596684904,"xid":7201,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.18}}
+{"database":"test","table":"product","type":"update","ts":1596684906,"xid":7216,"commit":true,"data":{"id":110,"name":"jacket","description":"new water resistent white wind breaker","weight":0.5},"old":{"description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"update","ts":1596684912,"xid":7235,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17},"old":{"weight":5.18}}
+{"database":"test","table":"product","type":"delete","ts":1596684914,"xid":7250,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":5.17},"old":{"weight":3.14}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"commit":true,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17},"old":{"weight":8.1}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"xoffset":0,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"commit":true,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}
diff --git a/pom.xml b/pom.xml
index b67e1e3..4c7792d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,8 +54,12 @@ under the License.
<flink.version>1.17.0</flink.version>
<flink-ci-tools.version>1.18-SNAPSHOT</flink-ci-tools.version>
<pulsar.version>3.0.0</pulsar.version>
+ <scala.binary.version>2.12</scala.binary.version>
<bouncycastle.version>1.69</bouncycastle.version>
+ <scala-reflect.version>2.12.7</scala-reflect.version>
+ <scala-library.version>2.12.7</scala-library.version>
+
<jsr305.version>1.3.9</jsr305.version>
<junit5.version>5.9.1</junit5.version>
<assertj.version>3.23.1</assertj.version>
@@ -421,6 +425,19 @@ under the License.
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
+
+ <!-- For dependency convergence -->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ <version>${scala-reflect.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala-library.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>